You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2017/10/18 18:42:39 UTC
[streams] branch gh-pages updated: empty gh-pages branch
This is an automated email from the ASF dual-hosted git repository.
sblackmon pushed a commit to branch gh-pages
in repository https://gitbox.apache.org/repos/asf/streams.git
The following commit(s) were added to refs/heads/gh-pages by this push:
new 4385c0d empty gh-pages branch
4385c0d is described below
commit 4385c0db72eccb0e0dfba841dcb8e453bd10fa93
Author: Steve Blackmon @steveblackmon <sb...@apache.org>
AuthorDate: Wed Oct 18 13:42:30 2017 -0500
empty gh-pages branch
---
KEYS | 240 ----
LICENSE | 202 ---
NOTICE | 5 -
README.md | 16 -
pom.xml | 1476 --------------------
publish-rc.sh | 46 -
release.sh | 137 --
src/main/resources/architecture.dot | 62 -
src/main/resources/example.dot | 104 --
src/main/resources/images/streams_logo.jpg | Bin 56578 -> 0 bytes
src/main/resources/integration.dot | 76 -
src/main/resources/streams-java-checkstyle.xml | 227 ---
src/main/resources/streams-java-intellij.xml | 493 -------
src/site/custom/project-info-report.properties | 299 ----
src/site/markdown/architecture.md | 35 -
src/site/markdown/changelog.md | 560 --------
src/site/markdown/code-conventions.md | 74 -
src/site/markdown/concepts.md | 57 -
src/site/markdown/credentials/twitter.md | 24 -
src/site/markdown/dependency-info.md | 107 --
src/site/markdown/downloads.md | 10 -
src/site/markdown/faq.md | 138 --
src/site/markdown/index.md | 13 -
src/site/markdown/install/docker.md | 24 -
src/site/markdown/install/git.md | 15 -
src/site/markdown/install/java.md | 16 -
src/site/markdown/install/maven.md | 19 -
src/site/markdown/install/sbt.md | 17 -
src/site/markdown/integration.md | 7 -
src/site/markdown/release-setup.md | 74 -
src/site/markdown/release.md | 258 ----
src/site/markdown/services/cassandra.md | 34 -
src/site/markdown/services/elasticsearch.md | 35 -
src/site/markdown/services/mongo.md | 34 -
src/site/markdown/services/neo4j.md | 29 -
src/site/markdown/source-repository.md | 17 -
src/site/markdown/website.md | 189 ---
src/site/markdown/who.md | 12 -
src/site/site.xml | 124 --
streams-components/README.md | 14 -
streams-components/pom.xml | 72 -
streams-components/streams-converters/README.md | 8 -
streams-components/streams-converters/pom.xml | 137 --
.../converter/ActivityConverterProcessor.java | 107 --
.../streams/converter/ActivityConverterUtil.java | 260 ----
.../ActivityObjectConverterProcessor.java | 105 --
.../converter/ActivityObjectConverterUtil.java | 265 ----
.../streams/converter/BaseDocumentClassifier.java | 89 --
.../converter/BaseObjectNodeActivityConverter.java | 104 --
.../BaseObjectNodeActivityObjectConverter.java | 72 -
.../converter/BaseStringActivityConverter.java | 102 --
.../BaseStringActivityObjectConverter.java | 71 -
.../apache/streams/converter/FieldConstants.java | 32 -
.../streams/converter/HoconConverterProcessor.java | 94 --
.../streams/converter/HoconConverterUtil.java | 138 --
.../streams/converter/LineReadWriteUtil.java | 256 ----
.../streams/converter/TypeConverterProcessor.java | 102 --
.../streams/converter/TypeConverterUtil.java | 84 --
.../ActivityConverterProcessorConfiguration.json | 26 -
...ivityObjectConverterProcessorConfiguration.json | 26 -
.../converter/LineReadWriteConfiguration.json | 36 -
.../streams-converters/src/site/markdown/index.md | 8 -
.../test/BaseActivityConverterProcessorTest.java | 95 --
.../BaseActivityObjectConverterProcessorTest.java | 95 --
.../converter/test/CustomActivityConverter.java | 71 -
.../test/CustomActivityConverterProcessorTest.java | 104 --
.../test/CustomActivityObjectConverter.java | 56 -
...CustomActivityObjectConverterProcessorTest.java | 103 --
.../converter/test/CustomDocumentClassifier.java | 72 -
.../apache/streams/converter/test/CustomType.java | 38 -
.../test/HoconConverterProcessorTest.java | 112 --
.../streams/converter/test/HoconConverterTest.java | 95 --
.../NoConfigActivityConverterProcessorTest.java | 94 --
.../converter/test/TestLineReadWriteUtil.java | 113 --
.../converter/test/TypeConverterProcessorTest.java | 194 ---
.../src/test/resources/test1.conf | 17 -
.../src/test/resources/test2.conf | 19 -
.../src/test/resources/test3a.conf | 19 -
.../src/test/resources/test3b.conf | 17 -
.../src/test/resources/test4.conf | 19 -
streams-components/streams-filters/README.md | 8 -
streams-components/streams-filters/pom.xml | 97 --
.../streams/filters/VerbDefinitionDropFilter.java | 99 --
.../streams/filters/VerbDefinitionKeepFilter.java | 99 --
.../streams-filters/src/src/markdown/index.md | 8 -
.../filters/test/VerbDefinitionFilterTest.java | 259 ----
.../streams-filters/src/test/resources/actor.json | 9 -
.../streams-filters/src/test/resources/follow.json | 12 -
.../streams-filters/src/test/resources/object.json | 9 -
.../streams-filters/src/test/resources/post.json | 12 -
.../src/test/resources/provider.json | 9 -
.../src/test/resources/targetrequired.json | 12 -
streams-components/streams-http/README.md | 8 -
streams-components/streams-http/pom.xml | 159 ---
.../http/persist/SimpleHTTPPostPersistWriter.java | 240 ----
.../http/processor/SimpleHTTPGetProcessor.java | 295 ----
.../http/processor/SimpleHTTPPostProcessor.java | 293 ----
.../http/provider/SimpleHttpProvider.java | 352 -----
.../streams/components/http/HttpConfiguration.json | 61 -
.../http/HttpPersistWriterConfiguration.json | 18 -
.../http/HttpProcessorConfiguration.json | 31 -
.../components/http/HttpProviderConfiguration.json | 21 -
.../streams-http/src/site/markdown/index.md | 26 -
.../test/java/SimpleHTTPPostPersistWriterTest.java | 138 --
streams-config/README.md | 8 -
streams-config/pom.xml | 153 --
.../streams/config/ComponentConfigurator.java | 92 --
.../apache/streams/config/StreamsConfigurator.java | 101 --
.../main/jsonschema/ComponentConfiguration.json | 26 -
.../src/main/jsonschema/StreamsConfiguration.json | 48 -
streams-config/src/main/resources/reference.conf | 11 -
streams-config/src/site/markdown/index.md | 8 -
.../config/test/ComponentConfiguratorTest.java | 125 --
.../config/test/StreamsConfiguratorTest.java | 129 --
.../src/test/resources/componentTest.conf | 26 -
streams-config/src/test/resources/evenmore.conf | 16 -
streams-config/src/test/resources/more.conf | 16 -
streams-config/src/test/resources/testResolve.conf | 3 -
streams-contrib/README.md | 11 -
streams-contrib/pom.xml | 91 --
streams-contrib/streams-amazon-aws/pom.xml | 80 --
.../streams-persist-kinesis/README.md | 8 -
.../streams-persist-kinesis/pom.xml | 114 --
.../amazon/kinesis/KinesisPersistReader.java | 180 ---
.../amazon/kinesis/KinesisPersistReaderTask.java | 118 --
.../amazon/kinesis/KinesisPersistWriter.java | 143 --
.../amazon/kinesis/KinesisConfiguration.json | 33 -
.../amazon/kinesis/KinesisReaderConfiguration.json | 19 -
.../amazon/kinesis/KinesisWriterConfiguration.json | 16 -
.../src/main/resources/components.dot | 49 -
.../src/main/resources/kinesis-read.conf | 27 -
.../src/main/resources/reference.conf | 5 -
.../src/site/markdown/index.md | 23 -
.../streams-persist-s3/README.md | 8 -
.../streams-amazon-aws/streams-persist-s3/pom.xml | 144 --
.../streams/s3/S3ObjectInputStreamWrapper.java | 163 ---
.../apache/streams/s3/S3OutputStreamWrapper.java | 146 --
.../org/apache/streams/s3/S3PersistReader.java | 220 ---
.../org/apache/streams/s3/S3PersistReaderTask.java | 90 --
.../org/apache/streams/s3/S3PersistWriter.java | 310 ----
.../org/apache/streams/s3/S3Configuration.json | 41 -
.../apache/streams/s3/S3ReaderConfiguration.json | 17 -
.../apache/streams/s3/S3WriterConfiguration.json | 37 -
.../src/main/resources/components.dot | 50 -
.../src/main/resources/s3-read.conf | 27 -
.../streams-persist-s3/src/site/markdown/index.md | 23 -
.../org/apache/streams/s3/S3PersistWriterTest.java | 80 --
.../streams-persist-cassandra/README.md | 8 -
streams-contrib/streams-persist-cassandra/pom.xml | 231 ---
.../apache/streams/cassandra/CassandraClient.java | 160 ---
.../streams/cassandra/CassandraPersistReader.java | 284 ----
.../streams/cassandra/CassandraPersistWriter.java | 316 -----
.../streams/cassandra/CassandraConfiguration.json | 70 -
.../src/main/resources/components.dot | 50 -
.../src/main/resources/reference.conf | 13 -
.../src/site/markdown/cassandra.md | 36 -
.../src/site/markdown/index.md | 23 -
.../streams-persist-cassandra/src/site/site.xml | 25 -
.../streams/cassandra/test/CassandraPersistIT.java | 106 --
.../src/test/resources/CassandraPersistIT.conf | 27 -
.../src/test/resources/cassandra.conf | 25 -
streams-contrib/streams-persist-console/README.md | 16 -
streams-contrib/streams-persist-console/pom.xml | 85 --
.../streams/console/ConsolePersistReader.java | 117 --
.../streams/console/ConsolePersistWriter.java | 87 --
.../streams/console/ConsolePersistWriterTask.java | 61 -
.../src/main/resources/components.dot | 43 -
.../src/site/markdown/index.md | 17 -
.../streams-persist-elasticsearch/README.md | 8 -
.../streams-persist-elasticsearch/pom.xml | 294 ----
.../elasticsearch/ElasticsearchClientManager.java | 117 --
.../elasticsearch/ElasticsearchMetadataUtil.java | 214 ---
.../elasticsearch/ElasticsearchPersistDeleter.java | 122 --
.../elasticsearch/ElasticsearchPersistReader.java | 232 ---
.../elasticsearch/ElasticsearchPersistUpdater.java | 147 --
.../elasticsearch/ElasticsearchPersistWriter.java | 636 ---------
.../streams/elasticsearch/ElasticsearchQuery.java | 246 ----
.../DatumFromMetadataAsDocumentProcessor.java | 126 --
.../processor/DatumFromMetadataProcessor.java | 112 --
.../processor/DocumentToMetadataProcessor.java | 96 --
.../processor/MetadataFromDocumentProcessor.java | 130 --
.../processor/PercolateTagProcessor.java | 371 -----
.../elasticsearch/ElasticsearchConfiguration.json | 27 -
.../ElasticsearchReaderConfiguration.json | 36 -
.../ElasticsearchWriterConfiguration.json | 62 -
.../src/main/resources/components.dot | 60 -
.../src/main/resources/elasticsearch-read.conf | 35 -
.../src/main/resources/elasticsearch-write.conf | 26 -
.../src/main/resources/reference.conf | 13 -
.../src/site/markdown/elasticsearch.md | 35 -
.../src/site/markdown/index.md | 32 -
.../src/site/site.xml | 25 -
.../processor/PercolateTagProcessorTest.java | 51 -
.../test/DatumFromMetadataProcessorIT.java | 100 --
.../test/ElasticsearchParentChildUpdaterIT.java | 150 --
.../test/ElasticsearchParentChildWriterIT.java | 187 ---
.../test/ElasticsearchPersistUpdaterIT.java | 204 ---
.../test/ElasticsearchPersistWriterIT.java | 132 --
.../test/TestDocumentToMetadataProcessor.java | 75 -
.../test/TestMetadataFromDocumentProcessor.java | 130 --
.../test/resources/ActivityChildObjectParent.json | 21 -
.../resources/DatumFromMetadataProcessorIT.conf | 22 -
.../ElasticsearchParentChildUpdaterIT.conf | 23 -
.../ElasticsearchParentChildWriterIT.conf | 23 -
.../resources/ElasticsearchPersistUpdaterIT.conf | 23 -
.../resources/ElasticsearchPersistWriterIT.conf | 23 -
.../streams-persist-filebuffer/README.md | 8 -
streams-contrib/streams-persist-filebuffer/pom.xml | 141 --
.../filebuffer/FileBufferPersistReader.java | 183 ---
.../filebuffer/FileBufferPersistWriter.java | 121 --
.../streams/file/FileBufferConfiguration.json | 17 -
.../src/main/resources/components.dot | 49 -
.../src/main/resources/filebuffer.conf | 20 -
.../src/main/resources/reference.conf | 5 -
.../src/site/markdown/index.md | 23 -
.../filebuffer/test/FileBufferPersistIT.java | 119 --
.../filebuffer/test/TestFileBufferPersist.java | 77 -
streams-contrib/streams-persist-graph/README.md | 8 -
streams-contrib/streams-persist-graph/pom.xml | 169 ---
.../java/org/apache/streams/graph/GraphHelper.java | 42 -
.../org/apache/streams/graph/HttpGraphHelper.java | 36 -
.../org/apache/streams/graph/QueryGraphHelper.java | 46 -
.../streams/graph/GraphReaderConfiguration.json | 20 -
.../src/main/resources/components.dot | 50 -
.../src/main/resources/graph-read.conf | 38 -
.../src/main/resources/graph-write.conf | 38 -
.../src/site/markdown/index.md | 23 -
.../src/site/markdown/neo4j.md | 29 -
.../streams-persist-graph/src/site/site.xml | 25 -
.../src/test/resources/sampleReaderResult.json | 119 --
streams-contrib/streams-persist-hbase/README.md | 8 -
streams-contrib/streams-persist-hbase/pom.xml | 210 ---
.../apache/streams/hbase/HbasePersistReader.java | 171 ---
.../streams/hbase/HbasePersistReaderTask.java | 44 -
.../apache/streams/hbase/HbasePersistWriter.java | 233 ---
.../streams/hbase/HbasePersistWriterTask.java | 63 -
.../apache/streams/hbase/HbaseConfiguration.json | 44 -
.../src/main/resources/components.dot | 47 -
.../src/main/resources/hbase.conf | 29 -
.../src/main/resources/reference.properties | 10 -
.../src/site/markdown/index.md | 22 -
.../streams/hbase/test/HbasePersistReaderIT.java | 89 --
.../streams/hbase/test/HbasePersistWriterIT.java | 96 --
.../src/test/resources/HbasePersistReaderIT.conf | 26 -
.../src/test/resources/HbasePersistWriterIT.conf | 26 -
streams-contrib/streams-persist-hdfs/README.md | 8 -
streams-contrib/streams-persist-hdfs/pom.xml | 213 ---
.../org/apache/streams/hdfs/HdfsConstants.java | 31 -
.../apache/streams/hdfs/WebHdfsPersistReader.java | 296 ----
.../streams/hdfs/WebHdfsPersistReaderTask.java | 116 --
.../apache/streams/hdfs/WebHdfsPersistWriter.java | 328 -----
.../streams/hdfs/WebHdfsPersistWriterTask.java | 61 -
.../org/apache/streams/hdfs/HdfsConfiguration.json | 41 -
.../streams/hdfs/HdfsReaderConfiguration.json | 17 -
.../streams/hdfs/HdfsWriterConfiguration.json | 32 -
.../src/main/resources/components.dot | 50 -
.../src/main/resources/hdfs-read.conf | 25 -
.../src/main/resources/hdfs-write.conf | 25 -
.../src/site/markdown/index.md | 23 -
.../streams/hdfs/test/HdfsPersistConfigTest.java | 167 ---
.../apache/streams/hdfs/test/TestHdfsPersist.java | 126 --
.../src/test/resources/hdfs-read.conf | 26 -
.../src/test/resources/hdfs-write.conf | 24 -
streams-contrib/streams-persist-kafka/README.md | 8 -
streams-contrib/streams-persist-kafka/pom.xml | 164 ---
.../apache/streams/kafka/KafkaPersistReader.java | 169 ---
.../streams/kafka/KafkaPersistReaderTask.java | 65 -
.../apache/streams/kafka/KafkaPersistWriter.java | 146 --
.../streams/kafka/KafkaPersistWriterTask.java | 63 -
.../apache/streams/kafka/KafkaConfiguration.json | 28 -
.../src/main/resources/components.dot | 49 -
.../src/main/resources/kafka.conf | 23 -
.../src/main/resources/reference.properties | 43 -
.../src/site/markdown/index.md | 23 -
streams-contrib/streams-persist-mongo/README.md | 8 -
streams-contrib/streams-persist-mongo/pom.xml | 219 ---
.../apache/streams/mongo/MongoPersistReader.java | 301 ----
.../apache/streams/mongo/MongoPersistWriter.java | 265 ----
.../apache/streams/mongo/MongoConfiguration.json | 36 -
.../src/main/resources/components.dot | 50 -
.../src/main/resources/mongo.conf | 23 -
.../src/site/markdown/index.md | 23 -
.../src/site/markdown/mongo.md | 34 -
.../streams-persist-mongo/src/site/site.xml | 25 -
.../apache/streams/mongo/test/MongoPersistIT.java | 112 --
.../src/test/resources/MongoPersistIT.conf | 23 -
streams-contrib/streams-persist-neo4j/pom.xml | 260 ----
.../streams/neo4j/CypherQueryGraphHelper.java | 344 -----
.../org/apache/streams/neo4j/Neo4jPersistUtil.java | 169 ---
.../apache/streams/neo4j/bolt/Neo4jBoltClient.java | 110 --
.../streams/neo4j/bolt/Neo4jBoltPersistReader.java | 326 -----
.../streams/neo4j/bolt/Neo4jBoltPersistWriter.java | 95 --
.../apache/streams/neo4j/http/Neo4jHttpClient.java | 92 --
.../streams/neo4j/http/Neo4jHttpGraphHelper.java | 104 --
.../streams/neo4j/http/Neo4jHttpPersistReader.java | 173 ---
.../streams/neo4j/http/Neo4jHttpPersistWriter.java | 171 ---
.../apache/streams/neo4j/CypherQueryResponse.json | 43 -
.../apache/streams/neo4j/Neo4jConfiguration.json | 30 -
.../streams/neo4j/Neo4jReaderConfiguration.json | 20 -
.../streams/neo4j/test/Neo4jBoltPersistIT.java | 156 ---
.../streams/neo4j/test/Neo4jHttpPersistIT.java | 138 --
.../neo4j/test/TestCypherQueryGraphHelper.java | 150 --
.../src/test/resources/Neo4jBoltPersistIT.conf | 20 -
.../src/test/resources/Neo4jHttpPersistIT.conf | 20 -
streams-contrib/streams-persist-riak/pom.xml | 229 ---
.../streams/riak/binary/RiakBinaryClient.java | 97 --
.../riak/binary/RiakBinaryPersistReader.java | 141 --
.../riak/binary/RiakBinaryPersistWriter.java | 159 ---
.../apache/streams/riak/http/RiakHttpClient.java | 105 --
.../streams/riak/http/RiakHttpPersistReader.java | 199 ---
.../streams/riak/http/RiakHttpPersistWriter.java | 166 ---
.../src/main/jsonschema/RiakConfiguration.json | 40 -
.../streams/riak/test/RiakBinaryPersistIT.java | 120 --
.../streams/riak/test/RiakHttpPersistIT.java | 115 --
.../src/test/resources/RiakBinaryPersistIT.conf | 21 -
.../src/test/resources/RiakHttpPersistIT.conf | 21 -
.../streams-processor-jackson/README.md | 8 -
streams-contrib/streams-processor-jackson/pom.xml | 94 --
.../CleanAdditionalPropertiesProcessor.java | 91 --
.../java/org/apache/streams/jackson/JsonUtil.java | 164 ---
.../streams/jackson/TypeConverterProcessor.java | 141 --
.../src/main/resources/components.dot | 46 -
.../src/site/markdown/index.md | 17 -
.../jackson/test/TypeConverterProcessorTest.java | 96 --
streams-contrib/streams-processor-json/README.md | 8 -
streams-contrib/streams-processor-json/pom.xml | 104 --
.../org/apache/streams/json/JsonPathExtractor.java | 157 ---
.../org/apache/streams/json/JsonPathFilter.java | 173 ---
.../src/main/resources/components.dot | 46 -
.../src/site/markdown/index.md | 17 -
.../streams/json/test/JsonPathExtractorTest.java | 102 --
.../src/test/resources/books.json | 21 -
.../streams-processor-peoplepattern/README.md | 8 -
.../streams-processor-peoplepattern/pom.xml | 140 --
.../peoplepattern/AccountTypeProcessor.java | 82 --
.../peoplepattern/DemographicsProcessor.java | 83 --
.../apache/streams/peoplepattern/AccountType.json | 30 -
.../apache/streams/peoplepattern/Demographics.json | 63 -
.../resources/templates/peoplepatternactor.json | 28 -
.../src/site/markdown/index.md | 15 -
streams-contrib/streams-processor-regex/README.md | 8 -
streams-contrib/streams-processor-regex/pom.xml | 57 -
.../regex/AbstractRegexExtensionExtractor.java | 137 --
.../streams/regex/RegexHashtagExtractor.java | 54 -
.../streams/regex/RegexMentionsExtractor.java | 56 -
.../apache/streams/regex/RegexUrlExtractor.java | 75 -
.../java/org/apache/streams/regex/RegexUtils.java | 103 --
.../src/main/resources/components.dot | 54 -
.../src/site/markdown/index.md | 19 -
.../streams/regex/RegexHashtagExtractorTest.java | 77 -
.../streams/regex/RegexMentionExtractorTest.java | 91 --
.../streams/regex/RegexUrlExtractorTest.java | 77 -
.../org/apache/streams/regex/RegexUtilsTest.java | 78 --
streams-contrib/streams-processor-urls/README.md | 8 -
streams-contrib/streams-processor-urls/pom.xml | 137 --
.../java/org/apache/streams/urls/LinkResolver.java | 450 ------
.../streams/urls/LinkResolverHelperFunctions.java | 165 ---
.../apache/streams/urls/LinkResolverProcessor.java | 124 --
.../org/apache/streams/urls/LinkDetails.json | 78 --
.../src/main/resources/components.dot | 42 -
.../src/site/markdown/index.md | 16 -
.../streams/urls/LinkHelperFunctionsTest.java | 168 ---
.../streams/urls/TestLinkUnwinderProcessor.java | 163 ---
.../streams-provider-facebook/README.md | 17 -
.../metadata/facebook_post.mup | 697 ---------
.../metadata/facebook_post.png | Bin 1038039 -> 0 bytes
streams-contrib/streams-provider-facebook/pom.xml | 180 ---
.../api/FacebookPageActivitySerializer.java | 67 -
.../api/FacebookPostActivitySerializer.java | 68 -
.../FacebookPublicFeedXmlActivitySerializer.java | 29 -
.../facebook/processor/FacebookTypeConverter.java | 227 ---
.../facebook/provider/FacebookDataCollector.java | 151 --
.../facebook/provider/FacebookEventClassifier.java | 67 -
.../provider/FacebookFriendFeedProvider.java | 330 -----
.../provider/FacebookFriendUpdatesProvider.java | 326 -----
.../facebook/provider/FacebookProvider.java | 163 ---
.../provider/FacebookUserInformationProvider.java | 328 -----
.../provider/FacebookUserstreamProvider.java | 348 -----
.../provider/page/FacebookPageDataCollector.java | 85 --
.../provider/page/FacebookPageProvider.java | 137 --
.../pagefeed/FacebookPageFeedDataCollector.java | 136 --
.../pagefeed/FacebookPageFeedProvider.java | 113 --
.../facebook/serializer/FacebookActivityUtil.java | 274 ----
.../serializer/FacebookStreamsPostSerializer.java | 89 --
.../streams/facebook/FacebookConfiguration.json | 85 --
.../FacebookPageProviderConfiguration.json | 20 -
.../FacebookUserInformationConfiguration.json | 25 -
.../facebook/FacebookUserstreamConfiguration.json | 25 -
.../org/apache/streams/facebook/Page.json | 281 ----
.../org/apache/streams/facebook/graph/Post.json | 207 ---
.../src/main/resources/components.dot | 56 -
.../src/main/resources/reference.conf | 134 --
.../src/site/markdown/index.md | 51 -
.../facebook/test/FacebookEventClassifierTest.java | 58 -
.../streams/facebook/test/SimplePageTest.java | 106 --
.../org/apache/streams/facebook/test/TestPage.java | 156 ---
.../test/data/FacebookActivityActorSerDeIT.java | 64 -
.../test/data/FacebookActivitySerDeIT.java | 73 -
.../facebook/test/data/FacebookPageSerDeIT.java | 76 -
.../facebook/test/data/FacebookPostSerDeIT.java | 97 --
.../test/providers/TestFacebookProvider.java | 100 --
.../providers/page/FacebookPageProviderIT.java | 74 -
.../pagefeed/FacebookPageFeedProviderIT.java | 70 -
.../test/resources/FacebookPageFeedProviderIT.conf | 23 -
.../src/test/resources/FacebookPageProviderIT.conf | 23 -
.../streams-provider-google/google-gmail/README.md | 14 -
.../streams-provider-google/google-gmail/pom.xml | 158 ---
.../gmail/provider/GMailImapProviderTask.java | 62 -
.../provider/GMailMessageActivitySerializer.java | 193 ---
.../streams/gmail/provider/GMailProvider.java | 183 ---
.../streams/google/gmail/GMailConfiguration.json | 17 -
.../google-gmail/src/main/resources/components.dot | 44 -
.../google-gmail/src/main/resources/gmail.conf | 6 -
.../google-gmail/src/site/markdown/index.md | 22 -
.../streams/gmail/test/GMailMessageSerDeTest.java | 67 -
.../streams-provider-google/google-gplus/README.md | 22 -
.../streams-provider-google/google-gplus/pom.xml | 238 ----
.../processor/GooglePlusCommentProcessor.java | 96 --
.../gplus/processor/GooglePlusTypeConverter.java | 140 --
.../gplus/provider/AbstractGPlusProvider.java | 260 ----
.../gplus/provider/GPlusActivitySerializer.java | 70 -
.../streams/gplus/provider/GPlusDataCollector.java | 68 -
.../gplus/provider/GPlusUserActivityCollector.java | 161 ---
.../gplus/provider/GPlusUserActivityProvider.java | 131 --
.../gplus/provider/GPlusUserDataCollector.java | 106 --
.../gplus/provider/GPlusUserDataProvider.java | 133 --
.../serializer/util/GPlusActivityDeserializer.java | 172 ---
.../serializer/util/GPlusCommentDeserializer.java | 103 --
.../serializer/util/GPlusEventClassifier.java | 68 -
.../serializer/util/GPlusPersonDeserializer.java | 122 --
.../serializer/util/GooglePlusActivityUtil.java | 298 ----
.../streams/google/gplus/GPlusConfiguration.json | 101 --
.../google-gplus/src/main/resources/components.dot | 54 -
.../google-gplus/src/main/resources/reference.conf | 13 -
.../google-gplus/src/site/markdown/index.md | 47 -
.../streams/gplus/GooglePlusCommentSerDeIT.java | 116 --
.../streams/gplus/GooglePlusPersonSerDeIT.java | 103 --
.../processors/GooglePlusActivitySerDeIT.java | 119 --
.../processors/GooglePlusTypeConverterIT.java | 132 --
.../providers/GPlusUserActivityProviderIT.java | 67 -
.../gplus/providers/GPlusUserDataProviderIT.java | 70 -
.../gplus/providers/TestAbstractGPlusProvider.java | 101 --
.../providers/TestGPlusUserActivityCollector.java | 302 ----
.../providers/TestGPlusUserDataCollector.java | 135 --
.../serializer/util/GPlusEventClassifierTest.java | 71 -
.../resources/GPlusUserActivityProviderIT.conf | 27 -
.../test/resources/GPlusUserDataProviderIT.conf | 27 -
streams-contrib/streams-provider-google/pom.xml | 68 -
.../streams-provider-instagram/README.md | 8 -
.../InstagramMediaDataActivitySerializer.mup | 897 ------------
.../metadata/instagram_to_activity_mapping.png | Bin 1123684 -> 0 bytes
streams-contrib/streams-provider-instagram/pom.xml | 226 ---
.../apache/streams/instagram/api/Instagram.java | 499 -------
.../instagram/api/InstagramOAuthRequestSigner.java | 141 --
.../instagram/api/InstagramRetryHandler.java | 60 -
.../org/apache/streams/instagram/api/Media.java | 81 --
.../org/apache/streams/instagram/api/Users.java | 85 --
.../processor/InstagramTypeConverter.java | 110 --
.../provider/InstagramAbstractProvider.java | 164 ---
.../instagram/provider/InstagramDataCollector.java | 120 --
.../instagram/provider/InstagramProviderUtil.java | 49 -
.../recentmedia/InstagramRecentMediaCollector.java | 123 --
.../recentmedia/InstagramRecentMediaProvider.java | 133 --
.../userinfo/InstagramUserInfoCollector.java | 87 --
.../userinfo/InstagramUserInfoProvider.java | 134 --
.../InstagramMediaFeedDataConverter.java | 77 -
.../serializer/InstagramUserInfoDataConverter.java | 89 --
.../serializer/util/InstagramActivityUtil.java | 358 -----
.../streams/instagram/api/CommentsResponse.json | 26 -
.../streams/instagram/api/MediaResponse.json | 23 -
.../org/apache/streams/instagram/api/Meta.json | 23 -
.../apache/streams/instagram/api/Pagination.json | 20 -
.../streams/instagram/api/RecentMediaResponse.json | 26 -
.../streams/instagram/api/SearchMediaRequest.json | 26 -
.../streams/instagram/api/SearchMediaResponse.json | 26 -
.../streams/instagram/api/SearchUsersRequest.json | 22 -
.../streams/instagram/api/SearchUsersResponse.json | 26 -
.../instagram/api/SelfLikedMediaRequest.json | 22 -
.../instagram/api/SelfRecentMediaRequest.json | 26 -
.../streams/instagram/api/UserInfoResponse.json | 23 -
.../instagram/api/UserRecentMediaRequest.json | 19 -
.../streams/instagram/api/UsersInfoResponse.json | 26 -
.../instagram/config/InstagramConfiguration.json | 64 -
.../InstagramRecentMediaProviderConfiguration.json | 20 -
.../InstagramUserInfoProviderConfiguration.json | 20 -
.../org/apache/streams/instagram/pojo/Comment.json | 26 -
.../org/apache/streams/instagram/pojo/Media.json | 164 ---
.../apache/streams/instagram/pojo/UserInfo.json | 47 -
.../src/main/resources/components.dot | 56 -
.../src/main/resources/reference.conf | 11 -
.../src/site/markdown/index.md | 50 -
.../test/api/InstagramOAuthRequestSignerTest.java | 75 -
.../data/InstagramMediaFeedDataConverterIT.java | 101 --
.../data/InstagramUserInfoDataConverterIT.java | 115 --
.../providers/InstagramRecentMediaProviderIT.java | 75 -
.../providers/InstagramUserInfoProviderIT.java | 72 -
.../resources/InstagramRecentMediaProviderIT.conf | 23 -
.../resources/InstagramUserInfoProviderIT.conf | 23 -
.../streams-provider-moreover/README.md | 8 -
streams-contrib/streams-provider-moreover/pom.xml | 190 ---
.../apache/streams/moreover/MoreoverClient.java | 121 --
.../moreover/MoreoverJsonActivitySerializer.java | 98 --
.../apache/streams/moreover/MoreoverProvider.java | 195 ---
.../streams/moreover/MoreoverProviderTask.java | 99 --
.../apache/streams/moreover/MoreoverResult.java | 200 ---
.../org/apache/streams/moreover/MoreoverUtils.java | 195 ---
.../moreover/MoreoverXmlActivitySerializer.java | 106 --
.../org/apache/streams/moreover/Moreover.json | 337 -----
.../streams/moreover/MoreoverConfiguration.json | 38 -
.../src/main/resources/components.dot | 54 -
.../src/main/xmlschema/com/moreover/articles10.xsd | 383 -----
.../src/main/xmlschema/com/moreover/binding.xjb | 33 -
.../src/site/markdown/index.md | 25 -
.../apache/streams/moreover/MoreoverTestUtil.java | 53 -
.../test/MoreoverJsonActivitySerializerIT.java | 81 --
.../test/MoreoverXmlActivitySerializerIT.java | 65 -
.../moreover/test/provider/MoreoverProviderIT.java | 64 -
.../src/test/resources/moreover.conf | 25 -
streams-contrib/streams-provider-rss/README.md | 8 -
streams-contrib/streams-provider-rss/pom.xml | 205 ---
.../streams/rss/processor/RssTypeConverter.java | 80 --
.../streams/rss/provider/RssEventProcessor.java | 118 --
.../streams/rss/provider/RssStreamProvider.java | 215 ---
.../rss/provider/RssStreamProviderTask.java | 253 ----
.../rss/provider/perpetual/RssFeedScheduler.java | 124 --
.../serializer/SyndEntryActivitySerializer.java | 240 ----
.../rss/serializer/SyndEntrySerializer.java | 327 -----
.../main/jsonschema/RssStreamConfiguration.json | 31 -
.../src/main/resources/components.dot | 51 -
.../src/site/markdown/index.md | 27 -
.../rss/provider/RssStreamProviderTaskIT.java | 139 --
.../rss/provider/RssStreamProviderTest.java | 125 --
.../provider/perpetual/RssFeedSchedulerTest.java | 96 --
.../streams/rss/test/RssStreamProviderIT.java | 115 --
.../streams/rss/test/RssTypeConverterTest.java | 35 -
.../rss/test/SyndEntryActivitySerializerIT.java | 133 --
.../src/test/resources/rss.conf | 32 -
.../src/test/resources/top100.txt | 100 --
streams-contrib/streams-provider-sysomos/README.md | 8 -
streams-contrib/streams-provider-sysomos/pom.xml | 189 ---
.../apache/streams/sysomos/SysomosException.java | 63 -
.../conversion/SysomosBeatActivityConverter.java | 151 --
.../apache/streams/sysomos/data/HeartbeatInfo.java | 131 --
.../streams/sysomos/data/SysomosTagDefinition.java | 101 --
.../sysomos/processor/SysomosTypeConverter.java | 66 -
.../sysomos/provider/AbstractRequestBuilder.java | 59 -
.../sysomos/provider/ContentRequestBuilder.java | 137 --
.../streams/sysomos/provider/RequestBuilder.java | 97 --
.../streams/sysomos/provider/SysomosClient.java | 44 -
.../sysomos/provider/SysomosHeartbeatStream.java | 258 ----
.../streams/sysomos/provider/SysomosProvider.java | 383 -----
.../apache/streams/sysomos/util/SysomosUtils.java | 89 --
.../org/apache/streams/sysomos/Sysomos.json | 61 -
.../streams/sysomos/SysomosConfiguration.json | 42 -
.../src/main/resources/components.dot | 51 -
.../src/main/resources/reference.conf | 5 -
.../src/main/xmlschema/com/sysomos/binding.xjb | 33 -
.../src/main/xmlschema/com/sysomos/sysomos.xsd | 91 --
.../src/site/markdown/index.md | 25 -
.../streams/sysomos/test/SysomosJsonSerDeIT.java | 69 -
.../streams/sysomos/test/SysomosXmlSerDeIT.java | 102 --
.../sysomos/test/provider/SysomosProviderIT.java | 65 -
.../src/test/resources/sysomos.conf | 27 -
streams-contrib/streams-provider-twitter/README.md | 8 -
streams-contrib/streams-provider-twitter/pom.xml | 222 ---
.../org/apache/streams/twitter/api/Account.java | 54 -
.../streams/twitter/api/AccountActivity.java | 125 --
.../apache/streams/twitter/api/DirectMessages.java | 70 -
.../org/apache/streams/twitter/api/Favorites.java | 50 -
.../org/apache/streams/twitter/api/Followers.java | 59 -
.../org/apache/streams/twitter/api/Friends.java | 59 -
.../apache/streams/twitter/api/Friendships.java | 105 --
.../java/org/apache/streams/twitter/api/Geo.java | 72 -
.../java/org/apache/streams/twitter/api/Media.java | 30 -
.../org/apache/streams/twitter/api/Statuses.java | 106 --
.../org/apache/streams/twitter/api/Twitter.java | 395 ------
.../api/TwitterOAuthRequestInterceptor.java | 153 --
.../streams/twitter/api/TwitterRetryHandler.java | 90 --
.../streams/twitter/api/TwitterSecurity.java | 132 --
.../java/org/apache/streams/twitter/api/Users.java | 62 -
.../streams/twitter/api/WelcomeMessageRules.java | 63 -
.../streams/twitter/api/WelcomeMessages.java | 59 -
.../twitter/converter/StreamsTwitterMapper.java | 104 --
.../twitter/converter/TwitterDateTimeFormat.java | 34 -
.../converter/TwitterDocumentClassifier.java | 92 --
.../converter/TwitterFollowActivityConverter.java | 87 --
.../twitter/converter/TwitterJodaDateSwap.java | 51 -
.../TwitterJsonDeleteActivityConverter.java | 88 --
.../TwitterJsonRetweetActivityConverter.java | 88 --
.../TwitterJsonTweetActivityConverter.java | 89 --
.../TwitterJsonUserActivityConverter.java | 87 --
.../TwitterJsonUserActivityObjectConverter.java | 59 -
...witterJsonUserstreameventActivityConverter.java | 131 --
.../converter/util/TwitterActivityUtil.java | 382 -----
.../processor/FetchAndReplaceTwitterProcessor.java | 172 ---
.../twitter/processor/TwitterTypeConverter.java | 34 -
.../twitter/processor/TwitterUrlApiProcessor.java | 89 --
.../provider/TwitterFollowersIdsProviderTask.java | 125 --
.../provider/TwitterFollowersListProviderTask.java | 121 --
.../twitter/provider/TwitterFollowingProvider.java | 367 -----
.../provider/TwitterFriendsIdsProviderTask.java | 126 --
.../provider/TwitterFriendsListProviderTask.java | 124 --
.../twitter/provider/TwitterProviderUtil.java | 48 -
.../twitter/provider/TwitterStreamHelper.java | 109 --
.../twitter/provider/TwitterStreamProvider.java | 361 -----
.../twitter/provider/TwitterTimelineProvider.java | 368 -----
.../provider/TwitterTimelineProviderTask.java | 110 --
.../provider/TwitterUserInformationProvider.java | 406 ------
.../TwitterUserInformationProviderTask.java | 77 -
.../twitter/api/AccountSettingsResponse.json | 108 --
.../streams/twitter/api/EventShowResponse.json | 19 -
.../streams/twitter/api/EventsListRequest.json | 24 -
.../streams/twitter/api/EventsListResponse.json | 22 -
.../streams/twitter/api/FavoritesListRequest.json | 41 -
.../streams/twitter/api/FollowersIdsRequest.json | 13 -
.../streams/twitter/api/FollowersIdsResponse.json | 32 -
.../streams/twitter/api/FollowersListRequest.json | 13 -
.../streams/twitter/api/FollowersListResponse.json | 33 -
.../streams/twitter/api/FollowingIdsRequest.json | 36 -
.../streams/twitter/api/FollowingListRequest.json | 41 -
.../streams/twitter/api/FriendsIdsRequest.json | 13 -
.../streams/twitter/api/FriendsIdsResponse.json | 32 -
.../streams/twitter/api/FriendsListRequest.json | 13 -
.../streams/twitter/api/FriendsListResponse.json | 33 -
.../twitter/api/FriendshipCreateRequest.json | 28 -
.../twitter/api/FriendshipDestroyRequest.json | 23 -
.../streams/twitter/api/FriendshipShowRequest.json | 16 -
.../twitter/api/FriendshipShowResponse.json | 33 -
.../twitter/api/FriendshipUpdateRequest.json | 33 -
.../twitter/api/FriendshipsLookupRequest.json | 30 -
.../twitter/api/FriendshipsOutgoingRequest.json | 23 -
.../streams/twitter/api/GeoSearchRequest.json | 52 -
.../streams/twitter/api/GeoSearchResponse.json | 57 -
.../streams/twitter/api/MediaStatusRequest.json | 24 -
.../streams/twitter/api/MediaStatusResponse.json | 56 -
.../streams/twitter/api/MessageCreateRequest.json | 19 -
.../apache/streams/twitter/api/ResponseErrors.json | 27 -
.../twitter/api/StatusesHomeTimelineRequest.json | 42 -
.../streams/twitter/api/StatusesLookupRequest.json | 35 -
.../api/StatusesMentionsTimelineRequest.json | 37 -
.../streams/twitter/api/StatusesShowRequest.json | 35 -
.../twitter/api/StatusesUserTimelineRequest.json | 57 -
.../streams/twitter/api/UsersLookupRequest.json | 35 -
.../streams/twitter/api/UsersShowRequest.json | 27 -
.../org/apache/streams/twitter/api/Webhook.json | 27 -
.../twitter/api/WelcomeMessageNewRequest.json | 23 -
.../twitter/api/WelcomeMessageNewResponse.json | 17 -
.../twitter/api/WelcomeMessageNewRuleRequest.json | 25 -
.../twitter/api/WelcomeMessageNewRuleResponse.json | 35 -
.../api/WelcomeMessageRulesListRequest.json | 24 -
.../api/WelcomeMessageRulesListResponse.json | 23 -
.../twitter/api/WelcomeMessagesListRequest.json | 24 -
.../twitter/api/WelcomeMessagesListResponse.json | 23 -
.../twitter/config/TwitterConfiguration.json | 84 --
.../config/TwitterFollowingConfiguration.json | 33 -
.../twitter/config/TwitterStreamConfiguration.json | 45 -
.../TwitterTimelineProviderConfiguration.json | 23 -
.../TwitterUserInformationConfiguration.json | 25 -
.../streams/twitter/pojo/AccountSettings.json | 111 --
.../apache/streams/twitter/pojo/CallToAction.json | 34 -
.../org/apache/streams/twitter/pojo/Delete.json | 37 -
.../streams/twitter/pojo/DirectMessageEvent.json | 61 -
.../org/apache/streams/twitter/pojo/Follow.json | 18 -
.../apache/streams/twitter/pojo/FriendList.json | 24 -
.../apache/streams/twitter/pojo/Friendship.json | 39 -
.../org/apache/streams/twitter/pojo/MediaItem.json | 16 -
.../streams/twitter/pojo/MessageAttachment.json | 19 -
.../apache/streams/twitter/pojo/MessageData.json | 31 -
.../org/apache/streams/twitter/pojo/Place.json | 99 --
.../apache/streams/twitter/pojo/QuickReply.json | 117 --
.../apache/streams/twitter/pojo/Relationship.json | 19 -
.../streams/twitter/pojo/RelationshipSource.json | 49 -
.../streams/twitter/pojo/RelationshipTarget.json | 27 -
.../org/apache/streams/twitter/pojo/Retweet.json | 19 -
.../org/apache/streams/twitter/pojo/User.json | 129 --
.../streams/twitter/pojo/UserstreamEvent.json | 52 -
.../apache/streams/twitter/pojo/WebhookEvents.json | 25 -
.../streams/twitter/pojo/WelcomeMessage.json | 23 -
.../streams/twitter/pojo/WelcomeMessageRule.json | 23 -
.../org/apache/streams/twitter/pojo/tweet.json | 219 ---
.../src/main/resources/components.dot | 63 -
.../src/main/resources/followers.conf | 14 -
.../src/main/resources/friends.conf | 14 -
.../src/main/resources/reference.conf | 18 -
.../src/main/resources/sample.conf | 17 -
.../src/main/resources/userinfo.conf | 14 -
.../src/main/resources/userstream.conf | 11 -
.../src/site/markdown/index.md | 54 -
.../src/site/markdown/twitter.md | 24 -
.../streams-provider-twitter/src/site/site.xml | 25 -
.../apache/streams/twitter/test/api/TwitterIT.java | 292 ----
.../twitter/test/api/TwitterSecurityTest.java | 159 ---
.../twitter/test/data/TwitterObjectMapperIT.java | 143 --
.../test/providers/TwitterFollowingProviderIT.java | 104 --
.../test/providers/TwitterStreamProviderIT.java | 74 -
.../test/providers/TwitterTimelineProviderIT.java | 70 -
.../TwitterUserInformationProviderIT.java | 73 -
.../test/utils/TwitterActivityConvertersTest.java | 140 --
.../TwitterActivityObjectsConvertersTest.java | 77 -
.../test/utils/TwitterDocumentClassifierTest.java | 94 --
.../test/resources/TwitterFollowingProviderIT.conf | 24 -
.../src/test/resources/TwitterIT.conf | 20 -
.../src/test/resources/TwitterObjectMapperIT.conf | 17 -
.../test/resources/TwitterStreamProviderIT.conf | 23 -
.../test/resources/TwitterTimelineProviderIT.conf | 21 -
.../TwitterUserInformationProviderIT.conf | 1018 --------------
streams-contrib/streams-provider-youtube/README.md | 8 -
streams-contrib/streams-provider-youtube/pom.xml | 184 ---
.../youtube/processor/YoutubeTypeConverter.java | 129 --
.../provider/YoutubeChannelDataCollector.java | 124 --
.../youtube/provider/YoutubeChannelProvider.java | 124 --
.../youtube/provider/YoutubeDataCollector.java | 70 -
.../streams/youtube/provider/YoutubeProvider.java | 270 ----
.../provider/YoutubeUserActivityCollector.java | 238 ----
.../provider/YoutubeUserActivityProvider.java | 132 --
.../youtube/serializer/YoutubeActivityUtil.java | 200 ---
.../serializer/YoutubeChannelDeserializer.java | 152 --
.../youtube/serializer/YoutubeEventClassifier.java | 62 -
.../serializer/YoutubeVideoDeserializer.java | 117 --
.../streams/youtube/YoutubeConfiguration.json | 105 --
.../src/main/resources/components.dot | 56 -
.../src/site/markdown/index.md | 50 -
.../processor/YoutubeTypeConverterTest.java | 111 --
.../provider/YoutubeChannelDataCollectorTest.java | 102 --
.../youtube/provider/YoutubeProviderTest.java | 166 ---
.../provider/YoutubeUserActivityCollectorTest.java | 354 -----
.../serializer/YoutubeEventClassifierTest.java | 50 -
.../youtube/serializer/YoutubeVideoSerDeTest.java | 120 --
.../test/providers/YoutubeChannelProviderIT.java | 73 -
.../providers/YoutubeUserActivityProviderIT.java | 72 -
.../test/resources/YoutubeChannelProviderIT.conf | 22 -
.../resources/YoutubeUserActivityProviderIT.conf | 22 -
streams-core/README.md | 8 -
streams-core/pom.xml | 83 --
.../java/org/apache/streams/core/DatumStatus.java | 28 -
.../apache/streams/core/DatumStatusCountable.java | 30 -
.../apache/streams/core/DatumStatusCounter.java | 135 --
.../org/apache/streams/core/StreamBuilder.java | 120 --
.../java/org/apache/streams/core/StreamsDatum.java | 167 ---
.../org/apache/streams/core/StreamsOperation.java | 46 -
.../apache/streams/core/StreamsPersistReader.java | 36 -
.../apache/streams/core/StreamsPersistWriter.java | 32 -
.../org/apache/streams/core/StreamsProcessor.java | 33 -
.../org/apache/streams/core/StreamsProvider.java | 63 -
.../org/apache/streams/core/StreamsResultSet.java | 63 -
.../org/apache/streams/core/util/DatumUtils.java | 60 -
streams-core/src/site/markdown/index.md | 8 -
streams-monitoring/README.md | 8 -
streams-monitoring/pom.xml | 138 --
.../jackson/DatumStatusCounterDeserializer.java | 81 --
.../streams/jackson/MemoryUsageDeserializer.java | 84 --
.../jackson/StreamsTaskCounterDeserializer.java | 96 --
.../jackson/ThroughputQueueDeserializer.java | 92 --
.../monitoring/persist/MessagePersister.java | 35 -
.../persist/impl/BroadcastMessagePersister.java | 97 --
.../persist/impl/LogstashUdpMessagePersister.java | 107 --
.../persist/impl/Slf4jMessagePersister.java | 49 -
.../monitoring/tasks/BroadcastMonitorThread.java | 221 ---
.../local/monitoring/MonitoringConfiguration.json | 23 -
.../org/apache/streams/pojo/json/Broadcast.json | 26 -
.../pojo/json/DatumStatusCounterBroadcast.json | 29 -
.../streams/pojo/json/MemoryUsageBroadcast.json | 37 -
.../pojo/json/StreamsTaskCounterBroadcast.json | 45 -
.../pojo/json/ThroughputQueueBroadcast.json | 45 -
.../src/main/resources/reference.conf | 3 -
streams-monitoring/src/site/markdown/index.md | 8 -
.../jackson/MemoryUsageDeserializerTest.java | 82 --
.../impl/BroadcastMessagePersisterTest.java | 61 -
.../impl/LogstashUdpMessagePersisterTest.java | 79 --
.../tasks/BroadcastMonitorThreadTest.java | 74 -
.../src/test/resources/MemoryUsageObjects.json | 1 -
streams-plugins/pom.xml | 150 --
streams-plugins/streams-plugin-cassandra/pom.xml | 196 ---
.../StreamsCassandraGenerationConfig.java | 104 --
.../StreamsCassandraResourceGenerator.java | 409 ------
.../StreamsCassandraResourceGeneratorMojo.java | 96 --
.../src/site/markdown/index.md | 36 -
.../StreamsCassandraResourceGeneratorCLITest.java | 64 -
.../StreamsCassandraResourceGeneratorMojoIT.java | 87 --
.../StreamsCassandraResourceGeneratorTest.java | 89 --
.../resources/streams-plugin-cassandra/pom.xml | 143 --
.../streams-plugin-elasticsearch/pom.xml | 197 ---
.../StreamsElasticsearchGenerationConfig.java | 104 --
.../StreamsElasticsearchResourceGenerator.java | 419 ------
.../StreamsElasticsearchResourceGeneratorMojo.java | 92 --
.../src/site/markdown/index.md | 38 -
...reamsElasticsearchResourceGeneratorCLITest.java | 53 -
...treamsElasticsearchResourceGeneratorMojoIT.java | 62 -
.../StreamsElasticsearchResourceGeneratorTest.java | 77 -
.../resources/streams-plugin-elasticsearch/pom.xml | 143 --
streams-plugins/streams-plugin-hbase/pom.xml | 196 ---
.../hbase/StreamsHbaseGenerationConfig.java | 113 --
.../hbase/StreamsHbaseResourceGenerator.java | 233 ---
.../hbase/StreamsHbaseResourceGeneratorMojo.java | 98 --
.../src/site/markdown/index.md | 40 -
.../test/StreamsHbaseResourceGeneratorCLITest.java | 53 -
.../test/StreamsHbaseResourceGeneratorMojoIT.java | 73 -
.../test/StreamsHbaseResourceGeneratorTest.java | 78 --
.../test/resources/streams-plugin-hbase/pom.xml | 145 --
streams-plugins/streams-plugin-hive/pom.xml | 196 ---
.../plugins/hive/StreamsHiveGenerationConfig.java | 102 --
.../plugins/hive/StreamsHiveResourceGenerator.java | 351 -----
.../hive/StreamsHiveResourceGeneratorMojo.java | 90 --
.../streams-plugin-hive/src/site/markdown/index.md | 40 -
.../test/StreamsHiveResourceGeneratorCLITest.java | 53 -
.../test/StreamsHiveResourceGeneratorMojoIT.java | 73 -
.../test/StreamsHiveResourceGeneratorTest.java | 79 --
.../src/test/resources/Tweet.hql | 313 -----
.../src/test/resources/expected/activity.hql | 219 ---
.../src/test/resources/expected/collection.hql | 63 -
.../src/test/resources/expected/media_link.hql | 27 -
.../src/test/resources/expected/object.hql | 77 -
.../test/resources/expected/objectTypes/place.hql | 96 --
.../src/test/resources/expected/verbs/purchase.hql | 220 ---
.../src/test/resources/streams-plugin-hive/pom.xml | 144 --
streams-plugins/streams-plugin-pig/pom.xml | 196 ---
.../plugins/pig/StreamsPigGenerationConfig.java | 106 --
.../plugins/pig/StreamsPigResourceGenerator.java | 356 -----
.../pig/StreamsPigResourceGeneratorMojo.java | 88 --
.../streams-plugin-pig/src/site/markdown/index.md | 38 -
.../test/StreamsPigResourceGeneratorCLITest.java | 53 -
.../test/StreamsPigResourceGeneratorMojoIT.java | 73 -
.../pig/test/StreamsPigResourceGeneratorTest.java | 77 -
.../src/test/resources/expected/media_link.pig | 17 -
.../test/resources/expected/objectTypes/file.pig | 17 -
.../resources/expected/objectTypes/photo-album.pig | 17 -
.../src/test/resources/streams-plugin-pig/pom.xml | 144 --
streams-plugins/streams-plugin-pojo/pom.xml | 205 ---
.../streams/plugins/JuneauPojoAnnotator.java | 102 --
.../plugins/StreamsPojoGenerationConfig.java | 124 --
.../plugins/StreamsPojoSourceGenerator.java | 112 --
.../plugins/StreamsPojoSourceGeneratorMojo.java | 109 --
.../streams-plugin-pojo/src/site/markdown/index.md | 38 -
.../test/StreamsPojoSourceGeneratorCLITest.java | 57 -
.../test/StreamsPojoSourceGeneratorMojoIT.java | 83 --
.../test/StreamsPojoSourceGeneratorTest.java | 80 --
.../src/test/resources/streams-plugin-pojo/pom.xml | 199 ---
streams-plugins/streams-plugin-scala/pom.xml | 224 ---
.../plugins/StreamsScalaGenerationConfig.java | 58 -
.../plugins/StreamsScalaSourceGenerator.java | 422 ------
.../plugins/StreamsScalaSourceGeneratorMojo.java | 92 --
.../src/site/markdown/index.md | 36 -
.../test/StreamsScalaSourceGeneratorCLITest.java | 60 -
.../test/StreamsScalaSourceGeneratorMojoIT.java | 78 --
.../test/StreamsScalaSourceGeneratorTest.java | 83 --
.../test/resources/streams-plugin-scala/pom.xml | 117 --
streams-pojo-extensions/README.md | 8 -
streams-pojo-extensions/pom.xml | 67 -
.../streams/pojo/extensions/ExtensionUtil.java | 163 ---
.../src/main/jsonschema/activity.json | 51 -
.../src/main/jsonschema/object.json | 63 -
.../src/site/markdown/README.md | 8 -
.../pojo/extensions/test/ExtensionUtilTest.java | 48 -
streams-pojo/README.md | 8 -
streams-pojo/metadata/Serializer.mup | 374 -----
streams-pojo/metadata/serializer.png | Bin 604337 -> 0 bytes
streams-pojo/pom.xml | 264 ----
.../org/apache/streams/data/ActivityConverter.java | 93 --
.../streams/data/ActivityObjectConverter.java | 63 -
.../apache/streams/data/ActivitySerializer.java | 63 -
.../apache/streams/data/DocumentClassifier.java | 39 -
.../org/apache/streams/data/util/ActivityUtil.java | 157 ---
.../org/apache/streams/data/util/RFC3339Utils.java | 209 ---
.../exceptions/ActivityConversionException.java | 42 -
.../exceptions/ActivitySerializerException.java | 46 -
.../jackson/StreamsDateTimeDeserializer.java | 81 --
.../streams/jackson/StreamsDateTimeFormat.java | 31 -
.../streams/jackson/StreamsDateTimeSerializer.java | 45 -
.../streams/jackson/StreamsJacksonMapper.java | 160 ---
.../streams/jackson/StreamsJacksonModule.java | 80 --
.../streams/jackson/StreamsPeriodDeserializer.java | 41 -
.../streams/jackson/StreamsPeriodSerializer.java | 42 -
.../org/apache/streams/juneau/JodaDateSwap.java | 63 -
.../org/apache/streams/juneau/ListSerializer.java | 44 -
.../pojo/StreamsJacksonMapperConfiguration.json | 22 -
streams-pojo/src/site/markdown/index.md | 26 -
.../streams/pojo/test/ActivitySerDeTest.java | 105 --
.../pojo/test/CustomDateTimeFormatTest.java | 76 -
.../streams/pojo/test/DateTimeSerDeTest.java | 82 --
.../apache/streams/pojo/test/RFC3339UtilsTest.java | 228 ---
streams-runtimes/README.md | 11 -
streams-runtimes/pom.xml | 40 -
.../streams-runtime-dropwizard/README.md | 8 -
.../streams-runtime-dropwizard/pom.xml | 262 ----
.../streams/dropwizard/GenericWebhookResource.java | 280 ----
.../dropwizard/StreamDropwizardBuilder.java | 64 -
.../streams/dropwizard/StreamsApplication.java | 180 ---
.../dropwizard/StreamsDropwizardModule.java | 48 -
.../streams/dropwizard/GenericWebhookData.json | 35 -
.../dropwizard/StreamsDropwizardConfiguration.json | 13 -
.../src/site/markdown/index.md | 16 -
.../test/GenericWebhookResourceTest.java | 87 --
.../dropwizard/test/StreamsApplicationIT.java | 49 -
.../dropwizard/test/TestStreamsApplication.java | 27 -
.../src/test/resources/configuration.yml | 34 -
streams-runtimes/streams-runtime-local/README.md | 8 -
streams-runtimes/streams-runtime-local/pom.xml | 176 ---
.../local/builders/InvalidStreamException.java | 41 -
.../streams/local/builders/LocalStreamBuilder.java | 509 -------
.../streams/local/builders/StreamComponent.java | 309 ----
.../streams/local/counters/DatumStatusCounter.java | 87 --
.../local/counters/DatumStatusCounterMXBean.java | 43 -
.../streams/local/counters/StreamsTaskCounter.java | 165 ---
.../local/counters/StreamsTaskCounterMXBean.java | 63 -
...treamOnUnhandleThrowableThreadPoolExecutor.java | 67 -
.../streams/local/queues/ThroughputQueue.java | 486 -------
.../local/queues/ThroughputQueueMXBean.java | 64 -
.../streams/local/tasks/BaseStreamsTask.java | 218 ---
.../tasks/LocalStreamProcessMonitorThread.java | 94 --
.../local/tasks/StatusCounterMonitorRunnable.java | 25 -
.../local/tasks/StatusCounterMonitorThread.java | 79 --
.../streams/local/tasks/StreamsMergeTask.java | 90 --
.../local/tasks/StreamsPersistWriterTask.java | 178 ---
.../streams/local/tasks/StreamsProcessorTask.java | 181 ---
.../streams/local/tasks/StreamsProviderTask.java | 263 ----
.../apache/streams/local/tasks/StreamsTask.java | 83 --
.../main/jsonschema/LocalRuntimeConfiguration.json | 43 -
.../src/main/resources/reference.conf | 6 -
.../src/site/markdown/index.md | 12 -
.../local/builders/LocalStreamBuilderTest.java | 407 ------
.../local/builders/ToyLocalBuilderExample.java | 45 -
.../local/counters/DatumStatusCounterTest.java | 137 --
.../local/counters/StreamsTaskCounterTest.java | 164 ---
...OnUnhandledThrowableThreadPoolExecutorTest.java | 115 --
.../queues/ThroughputQueueMultiThreadTest.java | 325 -----
.../queues/ThroughputQueueSingleThreadTest.java | 257 ----
.../apache/streams/local/tasks/BasicTasksTest.java | 310 ----
.../local/tasks/StreamsProviderTaskTest.java | 164 ---
.../local/test/processors/DoNothingProcessor.java | 65 -
.../PassthroughDatumCounterProcessor.java | 112 --
.../local/test/processors/SlowProcessor.java | 59 -
.../test/providers/EmptyResultSetProvider.java | 74 -
.../test/providers/NumericMessageProvider.java | 109 --
.../local/test/writer/DatumCounterWriter.java | 113 --
.../streams/local/test/writer/DoNothingWriter.java | 53 -
.../streams/local/test/writer/SystemOutWriter.java | 53 -
.../component/ExpectedDatumsPersistWriter.java | 76 -
.../streams/test/component/FileReaderProvider.java | 101 --
.../test/component/StreamsDatumConverter.java | 31 -
.../test/component/StringToDocumentConverter.java | 33 -
.../component/tests/TestComponentsLocalStream.java | 53 -
.../tests/TestExpectedDatumsPersitWriter.java | 67 -
.../component/tests/TestFileReaderProvider.java | 65 -
.../src/test/resources/TestFile.txt | 4 -
streams-schemas/pom.xml | 40 -
.../streams-schema-activitystreams/pom.xml | 114 --
.../src/main/jsonschema/activity.json | 108 --
.../src/main/jsonschema/collection.json | 47 -
.../src/main/jsonschema/media_link.json | 34 -
.../src/main/jsonschema/object.json | 99 --
.../src/main/jsonschema/objectTypes/alert.json | 19 -
.../main/jsonschema/objectTypes/application.json | 19 -
.../src/main/jsonschema/objectTypes/article.json | 19 -
.../src/main/jsonschema/objectTypes/audio.json | 19 -
.../src/main/jsonschema/objectTypes/badge.json | 19 -
.../src/main/jsonschema/objectTypes/binary.json | 19 -
.../src/main/jsonschema/objectTypes/bookmark.json | 19 -
.../src/main/jsonschema/objectTypes/comment.json | 19 -
.../src/main/jsonschema/objectTypes/device.json | 19 -
.../src/main/jsonschema/objectTypes/event.json | 51 -
.../src/main/jsonschema/objectTypes/file.json | 25 -
.../src/main/jsonschema/objectTypes/folder.json | 19 -
.../src/main/jsonschema/objectTypes/game.json | 19 -
.../src/main/jsonschema/objectTypes/group.json | 19 -
.../src/main/jsonschema/objectTypes/image.json | 22 -
.../src/main/jsonschema/objectTypes/issue.json | 25 -
.../src/main/jsonschema/objectTypes/job.json | 19 -
.../src/main/jsonschema/objectTypes/list.json | 28 -
.../src/main/jsonschema/objectTypes/note.json | 19 -
.../src/main/jsonschema/objectTypes/offer.json | 19 -
.../main/jsonschema/objectTypes/organization.json | 19 -
.../src/main/jsonschema/objectTypes/page.json | 19 -
.../main/jsonschema/objectTypes/permission.json | 36 -
.../src/main/jsonschema/objectTypes/person.json | 25 -
.../main/jsonschema/objectTypes/photo-album.json | 19 -
.../src/main/jsonschema/objectTypes/photo.json | 23 -
.../src/main/jsonschema/objectTypes/place.json | 39 -
.../src/main/jsonschema/objectTypes/playlist.json | 19 -
.../src/main/jsonschema/objectTypes/process.json | 19 -
.../src/main/jsonschema/objectTypes/product.json | 25 -
.../src/main/jsonschema/objectTypes/property.json | 48 -
.../src/main/jsonschema/objectTypes/question.json | 28 -
.../src/main/jsonschema/objectTypes/review.json | 22 -
.../src/main/jsonschema/objectTypes/role.json | 25 -
.../src/main/jsonschema/objectTypes/service.json | 19 -
.../src/main/jsonschema/objectTypes/song.json | 19 -
.../src/main/jsonschema/objectTypes/status.json | 19 -
.../src/main/jsonschema/objectTypes/task.json | 40 -
.../src/main/jsonschema/objectTypes/team.json | 19 -
.../src/main/jsonschema/objectTypes/video.json | 19 -
.../src/main/jsonschema/verbs/accept.json | 24 -
.../src/main/jsonschema/verbs/access.json | 24 -
.../src/main/jsonschema/verbs/acknowledge.json | 24 -
.../src/main/jsonschema/verbs/add.json | 24 -
.../src/main/jsonschema/verbs/agree.json | 24 -
.../src/main/jsonschema/verbs/append.json | 24 -
.../src/main/jsonschema/verbs/approve.json | 24 -
.../src/main/jsonschema/verbs/archive.json | 24 -
.../src/main/jsonschema/verbs/assign.json | 24 -
.../src/main/jsonschema/verbs/at.json | 24 -
.../src/main/jsonschema/verbs/attach.json | 24 -
.../src/main/jsonschema/verbs/attend.json | 24 -
.../src/main/jsonschema/verbs/author.json | 24 -
.../src/main/jsonschema/verbs/authorize.json | 24 -
.../src/main/jsonschema/verbs/borrow.json | 24 -
.../src/main/jsonschema/verbs/build.json | 24 -
.../src/main/jsonschema/verbs/cancel.json | 24 -
.../src/main/jsonschema/verbs/checkin.json | 24 -
.../src/main/jsonschema/verbs/close.json | 24 -
.../src/main/jsonschema/verbs/complete.json | 24 -
.../src/main/jsonschema/verbs/confirm.json | 24 -
.../src/main/jsonschema/verbs/consume.json | 24 -
.../src/main/jsonschema/verbs/create.json | 24 -
.../src/main/jsonschema/verbs/delete.json | 24 -
.../src/main/jsonschema/verbs/deliver.json | 24 -
.../src/main/jsonschema/verbs/deny.json | 24 -
.../src/main/jsonschema/verbs/disagree.json | 24 -
.../src/main/jsonschema/verbs/dislike.json | 24 -
.../src/main/jsonschema/verbs/experience.json | 24 -
.../src/main/jsonschema/verbs/favorite.json | 24 -
.../src/main/jsonschema/verbs/find.json | 24 -
.../jsonschema/verbs/flag-as-inappropriate.json | 24 -
.../src/main/jsonschema/verbs/follow.json | 24 -
.../src/main/jsonschema/verbs/give.json | 24 -
.../src/main/jsonschema/verbs/host.json | 24 -
.../src/main/jsonschema/verbs/ignore.json | 24 -
.../src/main/jsonschema/verbs/insert.json | 24 -
.../src/main/jsonschema/verbs/install.json | 24 -
.../src/main/jsonschema/verbs/interact.json | 24 -
.../src/main/jsonschema/verbs/invite.json | 24 -
.../src/main/jsonschema/verbs/join.json | 24 -
.../src/main/jsonschema/verbs/leave.json | 24 -
.../src/main/jsonschema/verbs/like.json | 24 -
.../src/main/jsonschema/verbs/listen.json | 24 -
.../src/main/jsonschema/verbs/lose.json | 24 -
.../src/main/jsonschema/verbs/make-friend.json | 24 -
.../src/main/jsonschema/verbs/open.json | 24 -
.../src/main/jsonschema/verbs/play.json | 24 -
.../src/main/jsonschema/verbs/post.json | 24 -
.../src/main/jsonschema/verbs/present.json | 24 -
.../src/main/jsonschema/verbs/purchase.json | 24 -
.../src/main/jsonschema/verbs/qualify.json | 24 -
.../src/main/jsonschema/verbs/read.json | 24 -
.../src/main/jsonschema/verbs/receive.json | 24 -
.../src/main/jsonschema/verbs/reject.json | 24 -
.../src/main/jsonschema/verbs/remove-friend.json | 24 -
.../src/main/jsonschema/verbs/remove.json | 24 -
.../src/main/jsonschema/verbs/replace.json | 24 -
.../src/main/jsonschema/verbs/request-friend.json | 24 -
.../src/main/jsonschema/verbs/request.json | 24 -
.../src/main/jsonschema/verbs/resolve.json | 24 -
.../src/main/jsonschema/verbs/retract.json | 24 -
.../src/main/jsonschema/verbs/return.json | 24 -
.../src/main/jsonschema/verbs/rsvp-maybe.json | 24 -
.../src/main/jsonschema/verbs/rsvp-no.json | 24 -
.../src/main/jsonschema/verbs/rsvp-yes.json | 24 -
.../src/main/jsonschema/verbs/satisfy.json | 24 -
.../src/main/jsonschema/verbs/save.json | 24 -
.../src/main/jsonschema/verbs/schedule.json | 24 -
.../src/main/jsonschema/verbs/search.json | 24 -
.../src/main/jsonschema/verbs/sell.json | 24 -
.../src/main/jsonschema/verbs/send.json | 24 -
.../src/main/jsonschema/verbs/share.json | 24 -
.../src/main/jsonschema/verbs/sponsor.json | 24 -
.../src/main/jsonschema/verbs/start.json | 24 -
.../src/main/jsonschema/verbs/stop-following.json | 24 -
.../src/main/jsonschema/verbs/submit.json | 24 -
.../src/main/jsonschema/verbs/tag.json | 24 -
.../src/main/jsonschema/verbs/terminate.json | 24 -
.../src/main/jsonschema/verbs/tie.json | 24 -
.../src/main/jsonschema/verbs/unfavorite.json | 24 -
.../src/main/jsonschema/verbs/unlike.json | 24 -
.../src/main/jsonschema/verbs/unsatisfy.json | 24 -
.../src/main/jsonschema/verbs/unsave.json | 24 -
.../src/main/jsonschema/verbs/unshare.json | 24 -
.../src/main/jsonschema/verbs/update.json | 34 -
.../src/main/jsonschema/verbs/use.json | 24 -
.../src/main/jsonschema/verbs/watch.json | 24 -
.../src/main/jsonschema/verbs/win.json | 24 -
.../src/main/xmlschema/activity.xsd | 94 --
.../activitystreams/test/SchemaValidationTest.java | 89 --
.../src/test/resources/activities/accept.json | 16 -
.../src/test/resources/activities/access.json | 17 -
.../src/test/resources/activities/acknowledge.json | 16 -
.../src/test/resources/activities/add.json | 21 -
.../src/test/resources/activities/agree.json | 15 -
.../src/test/resources/activities/append.json | 16 -
.../src/test/resources/activities/approve.json | 20 -
.../src/test/resources/activities/archive.json | 15 -
.../src/test/resources/activities/assign.json | 20 -
.../src/test/resources/activities/at.json | 15 -
.../src/test/resources/activities/attach.json | 20 -
.../src/test/resources/activities/attend.json | 15 -
.../src/test/resources/activities/author.json | 15 -
.../src/test/resources/activities/authorize.json | 23 -
.../src/test/resources/activities/borrow.json | 21 -
.../src/test/resources/activities/build.json | 16 -
.../src/test/resources/activities/cancel.json | 16 -
.../src/test/resources/activities/checkin.json | 16 -
.../src/test/resources/activities/close.json | 16 -
.../src/test/resources/activities/complete.json | 16 -
.../src/test/resources/activities/confirm.json | 17 -
.../src/test/resources/activities/consume.json | 16 -
.../src/test/resources/activities/create.json | 15 -
.../src/test/resources/activities/delete.json | 16 -
.../src/test/resources/activities/deliver.json | 20 -
.../src/test/resources/activities/deny.json | 23 -
.../src/test/resources/activities/disagree.json | 30 -
.../src/test/resources/activities/dislike.json | 15 -
.../src/test/resources/activities/experience.json | 16 -
.../src/test/resources/activities/favorite.json | 15 -
.../src/test/resources/activities/find.json | 19 -
.../activities/flag-as-inappropriate.json | 24 -
.../src/test/resources/activities/follow.json | 15 -
.../src/test/resources/activities/give.json | 28 -
.../src/test/resources/activities/host.json | 15 -
.../src/test/resources/activities/ignore.json | 15 -
.../src/test/resources/activities/insert.json | 19 -
.../src/test/resources/activities/install.json | 18 -
.../src/test/resources/activities/interact.json | 16 -
.../src/test/resources/activities/invite.json | 19 -
.../src/test/resources/activities/join.json | 15 -
.../src/test/resources/activities/leave.json | 15 -
.../src/test/resources/activities/like.json | 22 -
.../src/test/resources/activities/listen.json | 15 -
.../src/test/resources/activities/lose.json | 15 -
.../src/test/resources/activities/make-friend.json | 15 -
.../src/test/resources/activities/open.json | 15 -
.../src/test/resources/activities/play.json | 15 -
.../src/test/resources/activities/post.json | 28 -
.../src/test/resources/activities/present.json | 15 -
.../src/test/resources/activities/purchase.json | 16 -
.../src/test/resources/activities/qualify.json | 15 -
.../src/test/resources/activities/read.json | 16 -
.../src/test/resources/activities/receive.json | 16 -
.../src/test/resources/activities/reject.json | 15 -
.../test/resources/activities/remove-friend.json | 15 -
.../src/test/resources/activities/remove.json | 23 -
.../src/test/resources/activities/replace.json | 19 -
.../test/resources/activities/request-friend.json | 16 -
.../src/test/resources/activities/request.json | 23 -
.../src/test/resources/activities/resolve.json | 15 -
.../src/test/resources/activities/retract.json | 26 -
.../src/test/resources/activities/return.json | 19 -
.../src/test/resources/activities/rsvp-maybe.json | 16 -
.../src/test/resources/activities/rsvp-no.json | 16 -
.../src/test/resources/activities/rsvp-yes.json | 16 -
.../src/test/resources/activities/satisfy.json | 21 -
.../src/test/resources/activities/save.json | 20 -
.../src/test/resources/activities/schedule.json | 15 -
.../src/test/resources/activities/search.json | 21 -
.../src/test/resources/activities/sell.json | 19 -
.../src/test/resources/activities/send.json | 19 -
.../src/test/resources/activities/share.json | 16 -
.../src/test/resources/activities/sponsor.json | 17 -
.../src/test/resources/activities/start.json | 15 -
.../test/resources/activities/stop-following.json | 15 -
.../src/test/resources/activities/submit.json | 15 -
.../src/test/resources/activities/tag.json | 19 -
.../src/test/resources/activities/terminate.json | 15 -
.../src/test/resources/activities/tie.json | 24 -
.../src/test/resources/activities/unfavorite.json | 15 -
.../src/test/resources/activities/unlike.json | 15 -
.../src/test/resources/activities/unsatisfy.json | 20 -
.../src/test/resources/activities/unsave.json | 15 -
.../src/test/resources/activities/unshare.json | 15 -
.../src/test/resources/activities/update.json | 15 -
.../src/test/resources/activities/use.json | 15 -
.../src/test/resources/activities/watch.json | 16 -
.../src/test/resources/activities/win.json | 15 -
.../streams-schema-activitystreams2/pom.xml | 106 --
.../w3c/activitystreams/test/ExamplesSerDeIT.java | 136 --
streams-testing/README.md | 8 -
streams-testing/pom.xml | 57 -
.../src/test/resources/log4j.properties | 32 -
.../src/test/resources/logback-test.xml | 45 -
streams-util/README.md | 8 -
streams-util/pom.xml | 179 ---
.../org/apache/streams/util/ComponentUtils.java | 129 --
.../java/org/apache/streams/util/GuidUtils.java | 54 -
.../java/org/apache/streams/util/PropertyUtil.java | 115 --
.../org/apache/streams/util/SerializationUtil.java | 91 --
.../requests/backoff/AbstractBackOffStrategy.java | 85 --
.../api/requests/backoff/BackOffException.java | 69 -
.../util/api/requests/backoff/BackOffStrategy.java | 53 -
.../backoff/impl/ConstantTimeBackOffStrategy.java | 50 -
.../backoff/impl/ExponentialBackOffStrategy.java | 46 -
.../backoff/impl/LinearTimeBackOffStrategy.java | 39 -
.../util/oauth/tokens/AbstractOauthToken.java | 34 -
.../tokens/tokenmanager/SimpleTokenManager.java | 51 -
.../tokenmanager/impl/BasicTokenManager.java | 90 --
.../org/apache/streams/util/schema/FieldType.java | 32 -
.../org/apache/streams/util/schema/FieldUtil.java | 57 -
.../org/apache/streams/util/schema/FileUtil.java | 128 --
.../streams/util/schema/GenerationConfig.java | 63 -
.../org/apache/streams/util/schema/Schema.java | 95 --
.../apache/streams/util/schema/SchemaStore.java | 61 -
.../streams/util/schema/SchemaStoreImpl.java | 402 ------
.../org/apache/streams/util/schema/SchemaUtil.java | 78 --
.../org/apache/streams/util/schema/UriUtil.java | 56 -
.../api/requests/backoff/BackOffStrategyTest.java | 87 --
.../backoff/ConstantTimeBackOffStrategyTest.java | 43 -
.../backoff/ExponentialBackOffStrategyTest.java | 42 -
.../backoff/LinearTimeBackOffStrategyTest.java | 41 -
.../streams/util/files/StreamsScannerUtil.java | 43 -
.../tokens/tokenmanager/TestBasicTokenManager.java | 200 ---
.../streams/util/schema/test/PropertyUtilTest.java | 53 -
.../util/schema/test/SchemaOrderingTest.java | 176 ---
.../streams/util/schema/test/SchemaStoreTest.java | 86 --
streams-verbs/README.md | 8 -
streams-verbs/pom.xml | 177 ---
.../verbs/ObjectCombinationGenericOrdering.java | 89 --
.../verbs/ObjectCombinationSpecificOrdering.java | 99 --
.../streams/verbs/VerbDefinitionMatchUtil.java | 67 -
.../streams/verbs/VerbDefinitionResolver.java | 132 --
.../streams/verbs/VerbDefinitionTemplateUtil.java | 84 --
.../src/main/jsonschema/verb-definition.json | 99 --
streams-verbs/src/main/resources/delete.json | 19 -
streams-verbs/src/main/resources/follow.json | 19 -
streams-verbs/src/main/resources/post.json | 19 -
streams-verbs/src/main/resources/share.json | 19 -
streams-verbs/src/site/markdown/index.md | 19 -
.../ObjectCombinationGenericOrderingTest.java | 41 -
.../ObjectCombinationSpecificOrderingTest.java | 51 -
.../streams/verbs/VerbDefinitionResolverTest.java | 112 --
.../streams/verbs/VerbDefinitionTemplateTest.java | 84 --
.../apache/streams/verbs/VerbDefinitionTest.java | 66 -
streams-verbs/src/test/resources/activities.txt | 4 -
streams-verbs/src/test/resources/actor.json | 9 -
streams-verbs/src/test/resources/do.json | 14 -
streams-verbs/src/test/resources/follow.json | 12 -
streams-verbs/src/test/resources/nofields.json | 15 -
streams-verbs/src/test/resources/object.json | 9 -
streams-verbs/src/test/resources/post.json | 15 -
streams-verbs/src/test/resources/provider.json | 9 -
1235 files changed, 102266 deletions(-)
diff --git a/KEYS b/KEYS
deleted file mode 100644
index 6bcd629..0000000
--- a/KEYS
+++ /dev/null
@@ -1,240 +0,0 @@
-This file contains the PGP keys of various developers.
-
-Users: pgp < KEYS
- gpg --import KEYS
-Developers:
- pgp -kxa <your name> and append it to this file.
- (pgpk -ll <your name> && pgpk -xa <your name>) >> this file.
- (gpg --list-sigs <your name>
- && gpg --armor --export <your name>) >> this file.
-
-pub 2048R/3F94A968 2013-01-07
-uid Jason Letourneau <le...@apache.org>
-sig 3 3F94A968 2013-01-07 Jason Letourneau <le...@apache.org>
-sub 2048R/020FBED0 2013-01-07
-sig 3F94A968 2013-01-07 Jason Letourneau <le...@apache.org>
-
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1.4.12 (Darwin)
-
-mQENBFDrCSQBCADrCEd8fjnwpYn6aW7caXtwcrduZKbRPVisa+K+V9K1cm2zNg7Z
-unOk5cvlyLE3l8m2324I2cFml/ing/NaK6PY2+xJg+c9EXQ73/u5ccMU7NYuMYEG
-bPK5FMmr6B8q0UhpNIVZ0S2ZV1mfROGaLGPkQPuP36sU+u6Z6R/CruT7f6V/GVJ8
-66HQ087Lb24JTi1j5boLQlAIEFtF15eGxB45LT6IklEKAVf/B2TJK2TWDkcB6p0/
-j1EglDcOIPXcGBHTco4xC/qIDjpqAWd2N6KTcHB8ZHgpAQrq0PIC0iOqxWMfzdvA
-CFEhCKdH99Kqut9HGIl5++jwDlo2ntZh/52XABEBAAG0KEphc29uIExldG91cm5l
-YXUgPGxldG91cm5lYXVAYXBhY2hlLm9yZz6JATgEEwECACIFAlDrCSQCGwMGCwkI
-BwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEN+dqCk/lKlo50gIAN4MyxP4RAbgirEK
-j3wN0LnC7yzofi6+PMT03E4XKV9tTDB4xNROjxDAly5N6KuKk7F/QleQKVdQEQL1
-7YKnYV45QK1oYn2x06IkNwCFw4dNDqn41Gd2QpJY5twKTVFf0o3Fd7QBDCfoiW01
-Ke8ymubN9bdKN/LjjmkiCslhrweLScySDxv5upaPZvNP8cHrk8MsoDJRtsuxUQ7e
-/kQaNLLTNSW8bXevft5IEKUVky5YbPnPSea0iOjMR9gYaOE3N+Vik1NjixjKe3NF
-TnbJNSobHUkmVcD6xaNcAlX+0J9kaNH6+abp119a18Bg47rtN1EVLJYNKk5u8MJz
-EHNqTk25AQ0EUOsJJAEIAKWqm/hH97xE/re7HBz4PcSeXaYbaV4gYR6crfQNEUUF
-4xguHLVK/cpQBfB1aZJp/oJdB1v3ervYiN+PCSYOriTAK1SOj0zbOqEsmLmIOy/w
-FKGEJocVbNfpMlFcLKSL+Eq4Zz7FTUNvLksF5nI9llA/2spGncHj7DU1w0tvExjI
-EIaglfEpVsFlmehC1BMdoIeTDYnarY78A5hZ5ydDxbVk78qYpRJMC9N5Ez4IAY8N
-QGXNgA8wTpb2y86oCkEgNRl2RuEhRce0SjEZ+euttVvxRpzqEusdsQJbs8Eg5wpF
-h8+ZF63fUPTyqscZtkCeHbcskjMykow0pfB90xNtROkAEQEAAYkBHwQYAQIACQUC
-UOsJJAIbDAAKCRDfnagpP5SpaFbgCADNR6m7BdNs9lnpfVTjmgA5Bxg3ZoCJilbk
-7OM3QxvdcoDnb0iXJEgoAIO1QaiK395eQmam/dHupzVcwI/86Q2eDWUTx4zcUYRv
-m2C6P5yiEuH2792rLJ1SJbMy7AA3ZzBPwobjLmujHu9nO6SdX4hbpH3/LW+Xe3zV
-k+sS4gxii4SYQWqQyBf27nZIVVxcBVVdB0KmFcw6IEV7/LVho/tBpKV1BUoDeZj1
-BrhHUb9vCD16k6NX+gjQhKgRmZuljnC87ruUxLDEqhmiViVEjzrusx5uLVK7FmzF
-fLsE3mi3F0I3eJVwUFXji+nIeZXszGWvstmz8ADWWyvkEblYBv7H
-=iVYc
------END PGP PUBLIC KEY BLOCK-----
-
-pub 4096R/68D51867 2014-12-11 Steve Blackmon <sb...@apache.org>
- Key fingerprint = F869 F111 9FCD 1952 E379 71CD F824 5949 68D5 1867
-sub 4096R/4EB79D36 2014-12-11
-
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: SKS 1.1.6
-Comment: Hostname: keys2.kfwebs.net
-
-mQINBFSKDjYBEADGXFvQYlRV/8cigZ9eAis1B8kyERiRtyjYfB7BWb7sfwao4cs2eoPLzQHz
-zz5AW61EabQHrcqKWPVIdQe2ag3MeOp/cpuuiAnc4x5ioJjTYwrTImu9uadfGw63gUe4ynVk
-Oer6XeI0YO8H7rsWw40PQle7XEN1i/+kiZyk/cmHXJdQG5GUsnXc+3gSVMbUmBlo6huNbHEG
-jWyQEa152HOZX4eS1W0YThJvVLPwrduDHx7G/Lma9cMxCH2Dz37/BaG7nji8kqS6jNomzYT3
-GDRWu9DtQ8n/uZ4WJo2HG8dqSiTpC261i8JYjgm+4Ic6cRK64jI7gdNehfaN6KIbj0T8ttZz
-HVGex2e7wYE2kDZ1LGiA4dOYWCgnRwpKBrE4sh7/HCUURkJouQVGsbejsJPpSC2isrU/t447
-fieIM+8e3F9S932XCabTTHqhY5qXAPV/5csb0PjHo3DmoDMhwaoC+VhyxeNctymsQKvKYdZl
-ugRRxmYYa2JF11GwMd+QM0YgwpOqIKna8OshXHAGisDEaS+HSr3wJ5VoKRV1KnysGSXSnrsM
-jV6pb9ZmSdbaJnnAa3FZWLuguco97R8tnFfeZIXsc4AkywPPG75i9gyUsaUWm+ACsSxefhoY
-u1FFTSPOVGWAH3MAL0Noxk7PF6CfAJvH63fH3eNOKUYT8M8WFQARAQABtCVTdGV2ZSBCbGFj
-a21vbiA8c2JsYWNrbW9uQGFwYWNoZS5vcmc+iQIcBBABAgAGBQJUkj+1AAoJEBxXIVvuqkLA
-ixYP/26Bfm8v7eWhCoSoqMUcknIyZCvTYtqNedIhZX/6W3gX9kEVKSrsHdOxgKfrpk5KHb2P
-yuQ/INskrzSUvsokNbzgbHIy0qCT3/v3aFBVadhMpo2BCnn5ds3D4Qy4GF7wKarBsPGjjgqz
-/GgIT8Q8MhwOzCgo3jyUkmB7R69KwevEBNdtHXxqzMf7asEzv218pxOYDWkYO446dTckvzGN
-m2GVUlyjyLqLOb/zm8RcIH3tDTmNy3C4nOx76muX6WtV0io7nZukj+5LXZU6jjEHEYnKjxTX
-FQRXFkReAM7Tfk6NBCEVaHmGNrume9w6YQbLVsy3VKjgYDA7L2dlfOcrLZiNCp7nUdQ/CMx4
-XrS4WKnJotSQJ+ifEef+HBilfYT27B/hDibRgPVzNSBIpVtOoJyLkEziOd0m+IMLEly038Mn
-yV/17BscKmVsejqJM9N2ZS7FIx7I4qzk++vwNebNxXeekkBBSSSZ6cEDbifWWmoDgd8/w8Ni
-2fTSY4oxSeuHWWSJSnlH0cN2zZhYkoOB1axHlSh0gNFub2zaTAsgKfFeGpn0FQOecdbyjAU0
-AscHh0XYYoTm0PuwKgYQelYRL0mlRiexR8U2bVm88//ymrjPzCHvDdVLQwz76N5sYeQ8DYLi
-cb1AQuqtvhSNDMKcNc3P4IyPt2hq8RZr6ieMJwkBiQI4BBMBAgAiBQJUig42AhsDBgsJCAcD
-AgYVCAIJCgsEFgIDAQIeAQIXgAAKCRD4JFlJaNUYZ5J/EACNtHUh2+Q296uSuoa0zBAydrDp
-k9/jyebaom8b6INZDAAsHuqvowhij+xreUGf4dzw1/yetkYzmZCuUZKsfSC2kYukqH/XGgqz
-W1/OTJytI0mGc5GJkNsmp1KHv2b0lAbRRirk/HiAJm7PervhK7WhHygtUsXGBPNkXJ4p+bE9
-afH4g81YnIjK6+MdB1cKFEWpr3TZkGFg6fO4AW6DSlP8LelXOEbs1cydG/dtZdw+idtG3JUj
-NtfgSfjM/i3Dec8tAYym1SB73JBaokILbpVzRlz73iswVzbhWWIg4DkH3xunBrDGyksibOPr
-ABUOOR06CClnWe+h5ql8bGOJn0BNKSVi3nNEOJ1dxqNq69rUiY0L9xN0TVGvg3uEHP3zGtTs
-jlYfK08ClmIY6zXFaAunoT0vtZwCaSMvsnr8oIs8BgzqbDvBfRLIjlPzN8snucnNbq5V1Ki7
-VQ/rdBxJqcwYzJjKtTosdD2ux2nY9wmJQPASKjoKGP5xbUJ67VQYvyfGUifxncocAIiGPgtk
-tsxQnflqco26DbuF5SOWxvZK6TrjYCFjHRtTa2v7rlb9WBkN5IGCFRVAwoUaABO2tV+IXYz5
-B3ZzUfkW+17D36z9Lmy5vrjUmHbRWcz0PPuNN8Qel2zb5KUPm9Dp72ZkHeWaRKew7+ildPJ3
-qTVWi0xLXbkCDQRUig42ARAAo0P3b9ROnG3wxZJgWe9+4tP9NmjjLiH+ZoNb0ovX936rmii0
-KVAtsx8MO3QiwJ/lBaW9Fl/FEoIYDpUw+JqiyhePmQ0/klH3SZEQ05aYkB/c0phu0gFzM1RR
-Ka6MbzdVYADxW6KOlN7yGTetRZqPutSV53BeYO86ZB/M/vSgSmUULcbiBpk2YeN/QfuhjzJG
-zirM9ApNPgAJ+0UZbFdQyjSEnqhZI9eVuEtyiiLpFbnHobq/Wk/+1GM+lywqXqDLUXLTmAFY
-GbcB4UfVCG6lDI96pW0zWjZZZGR8HRqeYfGbiY7BkuS9s//caNHZ22Yb9jcdJFD4FJ2Q3RPh
-W8yCAKaHNAPx++iac/mfdNYY2DJZG7Zz0VEiJXAsTR6jl6D1hHFTTp3LIWE1upgbQ0Wh9UWw
-zo7VUR+Mj8n9xBkCQXLGUt9JRCB/YnoJVeNhr2J8ht4wF1hlBLLAh4+Zh0LXxXAvoCmeY24a
-Qr5iS1qLpowB8uVoGqa5dl7zu/ey+MojfyK3i6KQQOnIAR9xXoQdIA/rpguvy2WDXygQQvqp
-VbuAFMNX0jB95kWJmUTy1Ns4JvOxP1GJdJuToWHzgE5zhMxni/VO/50mK7ojKUugAoBUYqzV
-1tsCj63m/wf5aFr4yVFTqgxl5EgyvbUzA1hGF87Yo37Fp+RPqSglvMDo4N0AEQEAAYkCHwQY
-AQIACQUCVIoONgIbDAAKCRD4JFlJaNUYZ8kSD/9sWMJKPRm5aoCkz6scoqk4ZxMkW+kF8ddG
-mvx+sEkfeV7JMzC6wFKOHSInrWLwKdaM0meaQjXm3A5hchseQL/sgDmFeApiRizFrQQsTKQ1
-53yI3baujkrSFUUB6E/cZ6uJKfNu6SEQLZXGFQAlm+o4qaSciAYKFJZwRaD4g0FmMwaTB3pa
-bCe+ELiAIKvLt/SBFLBL0ynSH5DdUgPjhQYR+yf0yws6XLVlM59pGfzYUmipoLXz17fcAPyB
-ttZwtKuFXaDMQ6A25NoKDBCfKBzH6P/niuGhHjcPNXk58tvOH1nIGVLIg1Gh+Ljm92ZtOF2B
-eTkn/wFMBN9NeGLXeMgBuZ2ka+U8cmc2f0he/sXujfpR8uTB4CmkS5s7n+iStQ2eSFMUjIAq
-SivKnti1bigqWRRvLNsj/q/ITSONZOlC2lZJVCirXO6Dqv2A4+aL6K1EoS2IADQdIxT0GbzX
-/dPhPI0c7dsu09vXwLTKWMFDsS0mbSME2obPst7lc23K41MwjA7IxBndEbIPiY8R1AiebMvU
-cstlylPTuEkjRgs7dzlJ7wJP281et7JeAAp2y77Un7uA6BTgugK80fe0D0IB42D03L3TzlEi
-LAUn68YKEE1jzTvjEA9Tlak9SaQLxZK3D1QS/UzLyA3ejAeTrKqBHIE7Qiw0t5eqYMR1V16C
-Ng==
-=X9Je
------END PGP PUBLIC KEY BLOCK-----
-pub 4096R/D3541808 2014-01-09
-uid [ultimate] Suneel Marthi (CODE SIGNING KEY) <sm...@apache.org>
-sig 3 D3541808 2014-01-09 Suneel Marthi (CODE SIGNING KEY) <sm...@apache.org>
-sub 4096R/AF46E2DE 2014-01-09
-sig D3541808 2014-01-09 Suneel Marthi (CODE SIGNING KEY) <sm...@apache.org>
-
------BEGIN PGP PUBLIC KEY BLOCK-----
-Comment: GPGTools - https://gpgtools.org
-
-mQINBFLPJmEBEAC9d/dUZCXeyhB0fVGmJAjdjXfLebav4VqGdNZC+M1T9C3dcVsh
-X/JGme5bjJeIgVwiH5UsdNceYn1+hyxs8jXuRAWEWKP76gD+pNrp8Az0ZdBkJoAy
-zCywOPtJV2PCOz7+S5ri2nUA2+1Kgcu6IlSLMmYAGO0IAmRrjBEzxy9iGaxiNGTc
-LvQt/iVtIXWkKKI8yvpoJ8iFf3TGhpjgaC/h7cJP3zpy0SScmhJJASLXRsfocLv9
-sle6ndN9IPbDtRW8cL7Fk3VQlzp1ToVjmnQTyZZ6S1WafsjzCZ9hLN+k++o8VbvY
-v3icY6Sy0BKz0J6KwaxTkuZ6w1K7oUkVOQboKaWFIEdO+jwrEmU+Puyd8Np8jLnF
-Q0Y5GPfyMlqM3S/zaDm1t4D1eb5FLciStkxfg5wPVK6TkqB325KVD3aio5C7E7kt
-aQechHxaJXCQOtCtVY4X+L4iClnMSuk+hcSc8W8MYRTSVansItK0vI9eQZXMnpan
-w9/jk5rS4Gts1rHB7+kdjT3QRJmkyk6fEFT0fz5tfMC7N8waeEUhCaRW6lAoiqDW
-NW1h+0UGxJw+9YcGxBC0kkt3iofNOWQWmuf/BS3DHPKT7XV/YtBHe44wW0sF5L5P
-nfQUHpnA3pcZ0En6bXAvepKVZTNdOWWJqMyHV+436DA+33h45QL6lWb/GwARAQAB
-tDVTdW5lZWwgTWFydGhpIChDT0RFIFNJR05JTkcgS0VZKSA8c21hcnRoaUBhcGFj
-aGUub3JnPokCNwQTAQoAIQUCUs8mYQIbAwULCQgHAwUVCgkICwUWAgMBAAIeAQIX
-gAAKCRC08czE01QYCOKKEAChRtHBoYNTX+RZbFO0Kl1GlN+i1Ik0shEm5ZJ56XHv
-AnFx/gRK7CfZzJswWo7kf2s/dvJiFfs+rrolYVuO6E8gNhAaTEomSuvWQAMHdPcR
-9G5APRKCSkbZYugElqplEbSphk78FKoFO+sml52M7Pr9jj88ApBjoFVVY8njdnNq
-6DVlaDsg8YninCD78Z7PNFnRGwxyZ8Qd4Dh0rG+MUTfAWopZu6/MxpQxU7QpeVeX
-SIMLg7ClFrGfXnZcszYF4dnav1aa0i7W88PAdYNPko7tC5qz5yv2ep7t2gRbcYKf
-RXhYC2FHQey3wPhMKjA8V436lAqmfYnY/YdmhEy9Xq/1EdX1nHsQ7OEkfgXK14WM
-F+rnqXRAl/0cwiyb41eocdg5kpZFIKgCYT02usLWxwNnd3jOCe109Ze3y3acN/G8
-+xOf9YRfNVAe6pD8H6ieRbv9gRjBmsbz9bXQCmxFnDqxNri5Me6gBAQPNmYTJD0h
-jgJTK6o0vJ0pwjBLauasJsLu+1tR3Cb0dxPE+JVaTF26FCd7pM7W6KdVfod9ZfrN
-cSyJ/cECc2KvYVGmTjQNVo1dYG0awBachlWnYNt+0Qx4opLsczZOLtPKtFY4BJA7
-aZoXT4Qf9yB8km7x2/cgNExVbFummToJ/IP3M39/EaryspsQQuM5Qu5Q5lZp8Qnn
-ybkCDQRSzyZhARAA7bAawFzbJaghYnm6mTZyGG5hQmfAynbF6cPAE+g2SnXcNQjP
-6kjYx3tSpb7rEzmjQqs46ztqdec6PIVBMhakON6z27Zz+IviAtO/TcaZHWNuCAjw
-FXVQZ+tYsSeiKInttfkrQc8jXAHWwSkSjLqNpvQpBdBEX80MYkFB6ZPOeON2+/Ta
-GC1H/HU2YngF0qQSmG33KKG6ezihBJdKxU6t2tsQfTlCmZW6R6MGpS9fVurYMKBk
-vR+7RGZ/H6dSjWPcpxhusGg92J9uz7r5SopN1wSdyPMUCMAFGeyoxcAuBDl38quU
-H/ENG3x5LDPq2aEH2AJ6yvZfIXbeJ1zmXf2cAHv+HbmvZaTSp0XIjq8Yxh8NkYEC
-ZdfRWmsGLIpU16TkBijpK3Dn9MDXjHGT3V8/qfdpURtMvIaL8WFrq9ejcy/vGRFn
-mCYqxIIPH+vLiMXKWtuMc61GN3ES21msKQH6IuQxxfQLyhK44L/pv7FpF4E+6LaE
-8uRwAex5HIDpR1v4aJq089rRtye9VXTJJLZ7lYs0HctdZ30QbBRWT4jS9d9rj3cr
-HgQ7mIGO9TAfK2kWc6AJN/EvxPWNbOwptsTUzAF/adiy9ax8C18iw7nKczC+2eN6
-UcbxXiPdytuKYK7O9A8S9e1w89GwpxYN7Xfn2o6QfpSbL9cLKiinOeV+xikAEQEA
-AYkCHwQYAQoACQUCUs8mYQIbDAAKCRC08czE01QYCG7yD/471dmyOD+go8cZkdqR
-3CHhjH03odtI0EJNVy4VGEC0r9paz3BWYTy18LqWYkw3ygphOIU1r8/7QK3H5Ke3
-c4yCSUxaMk5SlAJ+iVRek5TABkR8+zI+ZN5pQtqRH+ya5JxV4F/Sx5Q3KWMzpvgY
-n6AgSSc3hEfkgdI7SalIeyLaLDWv+RFdGZ5JU5gD28C0G8BeH8L62x6sixZcqoGT
-oy9rwkjs45/ZmmvBZhd1wLvC/au8l2Ecou6O8+8m26W8Z7vCuGKxuWn0KV3DLLWe
-66uchDVlakGoMJSPIK06JWYUlE+gL0CW+U2ekt/v2qb8hGgMVET3CBAMq+bFWuJ6
-juX7hJd7wHtCFfjnFDDAkdp2IIIZAlBW6FZGv7pJ82xsW6pSAg0A7VrV6nTtMtDv
-T8esOfo/t4t0gaL7bivy9DVVdATbUBcJJFpoVoe5MxiyjptveqPzIRwzt04n52Ph
-ordVWAnX5AokXWTg+Glem/EWEuf7jUuZArfqCSl/sZoQdXGTjR7G4iFscispji4+
-kNjVQsItqFbgDpuc6n+GcFxlKQ7YMCnu5MVtTV01U4lFs0qy0NTUqsuR35DM4z14
-DkFmj1upWAayCoXTpKzsHBvJZPC+Wqf9Pl3O47apelg7KxU3S011YfXpVPvCTKBv
-kD2o/5GKWS5QkSUEUXXY1oDiLg==
-=f8kJ
------END PGP PUBLIC KEY BLOCK-----
-
-pub 4096R/C5F10B99 2016-12-29 [expires: 2017-12-31]
-uid [ultimate] Joey Frazee <jf...@apache.org>
-sig 3 C5F10B99 2017-01-02 Joey Frazee <jf...@apache.org>
-uid [ultimate] Joey Frazee <jo...@icloud.com>
-sig 3 C5F10B99 2017-01-02 Joey Frazee <jf...@apache.org>
-sub 4096R/C34484EF 2016-12-29 [expires: 2017-12-31]
-sig C5F10B99 2017-01-02 Joey Frazee <jf...@apache.org>
-
------BEGIN PGP PUBLIC KEY BLOCK-----
-
-mQINBFhlO8ABEACsfCjuAT1EVcAS8Wvhxq9vadj57nQhWvZmHttS8Vuvucqot9gt
-2VMbvihwWj0pEUfeGnvhdGyHvm1qHj+A4eR4PHZ6E53HEfV7IlUjoH+2gKFudYgz
-/QUMgtFjhBxVxdP+xY8LSq4KtfvCCrdHFb0k+3K+FgwFeTGUrTJM8+5Lgjgrq2Zy
-iICqNCdyK+rYOadCJP8CxxWjudrbkI+yXmIvYcRei8C5SyvBmfV4ozVcevhsMxOC
-6uMPG7dFj6AqD7lTy7BLmNhQ36oLs3FMmevIcjzwrOe6tF9wvGR4U+5Tvxfmb5cA
-8EA4wFjqPo6BxKA4ojUEhvuWlb4t6sqKDRwTm90GnBTD1jvG+uXXz3rszTb0MteY
-4oCsSw0h510jJgEfY1qFmNrHvA/ULzaQAVA6x2nm1Izq25atzBdeQbI9KzlBrZ8k
-PZdwQZSXFymgBGEQLoxnJQbq9uVh0cAbWUucuk7Ty0DydJWgX0M4GNOFy897euvk
-K/6nEJQDXIsAuvGwaNlmlKp6t0vdpPmCi96jvmAklZr54O4K5FDK5BfMsBbJ9Kn4
-N7LIrxJ1Eblqb7cUDJKeuGoWxWgPrWaTvDe0S4BV3rqzbQ2jwgehYqJjl2jZ1zQk
-ffVJOtLcblsgXjwdtoGr+xNg+Fi5ul2ZbURdxE2uOay752cGtSoNY7VaxwARAQAB
-tCBKb2V5IEZyYXplZSA8amZyYXplZUBhcGFjaGUub3JnPokCQgQTAQgALAIbAwcL
-CQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAhkBBQJYaskcBQkB5B3cAAoJEHjlJtLF
-8QuZBHUP/A/aD4nD1tYTJj91DmFEBNm92oXvkpin1jv2uXs6EBLkHyJnpD3GGcBS
-t2qsI3W7JPmxLFdQ84NwLJTUEbOxS8CgoaIYH5krv490ina9LH1B1aItY/qTh+6t
-lp98+GhNr1jRaYpZwkQYQa43CTsroiJo6NHGOQ0G2Ryi+SSDHnddfT0wsO4qttcA
-s9CLsmDcwJVog2vjHXBk9EdVjB5kmNNYZdd+whGnE3bl0oex+FJ7RSJu5dw/3J21
-yjU8jCI7PlZW30WTrWnqJOGpSkkONhdZQxp5FEgxefu3/E5E3lQomkaLpdHToXCc
-2KwRzZ6rD/cG+yDtVhWM2mIjCGovcCO3FMHPUbW0aEhzszypSM5V/zuZMLNggUrt
-er6iY63HO4KvSr6u8bLSra873I8T6VTfKTo3wjJLzTIH3v2NKRuKhKT3vuZA3Hh2
-kC267OFxrtiXAxuuvoWExP7dTWNlk+D3YrDfhZOKRpBR73R2gvn3ov59uCDEeEaR
-karZVmqXVoxN/A02zn5X+xBmHMqYajUG9gq4tdWVHvg+4SfzgV0byDAy+qvTRKRa
-Dc4ReeOUuOlrQZz4rjERSiRbqKFc0cN/yuoT4+big5uxEIHsbd5jZCMTILaGwb48
-gPt7/ok6WszHFJtLACmtFI7Runyz12mTHREA7E0MZGx0ywxhw0N2tCRKb2V5IEZy
-YXplZSA8am9leS5mcmF6ZWVAaWNsb3VkLmNvbT6JAj8EEwEIACkCGwMHCwkIBwMC
-AQYVCAIJCgsEFgIDAQIeAQIXgAUCWGrJLQUJAeQd3AAKCRB45SbSxfELmcpLEACd
-4CawnyZVKLo6pFKBRVEgPkYarhuKIqrHc8o0TKS/nZC1FvFQ/4aXz/n0Sz9pPikh
-9b8our3QzMB2WUPdKjCw8QT0rE/qOKy5NrYc9FXzeRgncrNsrGy/df/Tnv0h9kOg
-JxkhQU6MecDVwUJ+mgA5KL7RUo41msGQqtgfPid3wCW9rT+R8+oSvtaDpNbZ++P6
-+UxMWRc286LPbTapoagRw3zh5VTwHJ5qERMrSOZe9UE43KBtPyzOZt6ZEhA9By3B
-eek3haEUcYJIGrqmjlC8gNLy5LqwEmUApGJLv+5BnelaxCljkfLVR92ZRvZQX3/C
-1UB24iTQn4gDkGBZCG8JxWg5ZQlqjomTsIyzGb3um931XmhIQbVfdMCk51Zx61gW
-EbW10RyqnCQsCVaOeSfv5TCMkqgbGDHaQhkDA5wbOa721vOo+Jh1ZLBRMtQ0rnna
-rB3PvZd/AO1xYPhSJbsQNjNxaNsyVENRn4Phpvwe7fhMKVB2bFYamsP4b5ymhRAy
-480L8WnBZBRfVDr+qezk1K2Uk/cOwj9mX+3dRWTygtsGgPiRPDPbhxmxeACKZ5Vg
-FpFfmyLhdVv3pehpr8Hs5741FybvdDPpS+w2/HuxBGhz4wnsag3ZIlHV1n1DKHpw
-yyjdw/vVVXgolOi23u8p/OQ/86Uo+B5UTU2hanMUQ7kCDQRYZTvAARAAwglBhvUF
-Nm+8c8g4mc8QX3NL4T8X07LZtDBAbUGaImYq1cRa8Nh0Z/PLvTVxO2DqEniNQcBZ
-8aXhiWm0wtzyL7bRA5HADFRoB9MQacJY7INnmqq67Q7pYuLKu3/coRRNAbGERnz5
-DcY/i6vzJJ86gQmnThTq9Im4R3FM8xb5PFpaWR7l2yC2P2WNQANm0tV8EeOEdGxu
-JRZyCvFje+6AxdHlF6Vs1aKNP6r4GXJ2AAlvEf0rcJ632xdh2IPQYVI8nN0Njc8w
-KHZVnl2GhLq91ZOSN88U5fr8S1Qp6FzM8Ayh3hvLgKav2V+oRE4OPqFiYm+WnKqy
-eBpAd0MmhVlsAuB9MatdUBMy6qbBqgElyvOCf7fDMm15zBxz9s/4VchRJGBVco7E
-au98jzAv0OerdU9MYBvuHQ5LQKUnaJ9WH0c+tpFNQIQ9Cdw8VwZ3D2dPg2et1b61
-HYIAHRi/jSltJNoi9oOW9itahojcx4mPbvVkcaV0fREkItF1DVHBY/ewHBSDzfm1
-g41zzaMDQXhXYSnYpny01R+glZGmOnxkeUsh0QPtfgzZZYLDbzd8yWYMtsaIk6AR
-CBmu7ql/2FlI6tqOKh/PyAh2lrrimLV7CjnoRmfUdb8t1h8Jl6uYNQ2Kv4iGokwB
-q8MeNMg4gbpcXM/Lr+UQYgXEw0aATYHVsaMAEQEAAYkCJQQYAQgADwIbDAUCWGrJ
-RwUJAeQeBwAKCRB45SbSxfELmSb8EACpH9F/LU9cBDlfY9f2+4VUleJGzW9IO/6G
-X5skiCgjuwS8vFhmv+5nVUvvOz2K8rRzAo79OCsPaQ4bvIS92w+1hVCTn1eXpCRh
-mQc8bjd6GYBtBsXnEWr1X0/zwLPCvPEQeXj3nppTPED83j+d1CjVLUa89F2vRNZV
-ZHpkl7z3NzI+0ZcdTVIX5PgmCXH2NfVQ14I0ksewSIpCPOBlfRgbSoS3YCdcsbMf
-J3uQgBp8dkb047wLAPKXlIMuMfWsKj/qc9dejoYi1Jx6KFsEnSMfsG4+HI7NNGwo
-Y0JD4uU9MEPzCrwygh6n+VSGQj1GyOa88w3eR2pP6MoxOwtBwB/QH8tYXVLo9K7h
-F1OsCk4cEP7Nq1J5yPaLoZtHAjePuEwdAntYlHt0/TCgyU1Rw/A79LJ4MSQMJIeR
-+37tT4mVqFkpQNkRYPM/w5f9T/AwvsGVq/g7PiDvaocelDnzeT0pBmEDngG7hR3Z
-nPLZPKMjiVRbkwI8+1TAq4cLTgoQFXGf9CPjHeo2ckg/oll+GVYuH5jle4IKBSw8
-+iMANxsDczfRILl7zdE+Y3+3/AbGYK1O80ThPynDSLt4Vmat7UWB31gPKqVJiz7Q
-mg6L4tNgzy89/BzIgH8aqoJGe6+uE2QkEBKB8SgpOPRBPqMTS8yojdHfoKfGwme6
-TkzA3pbw+g==
-=FlWb
------END PGP PUBLIC KEY BLOCK-----
diff --git a/LICENSE b/LICENSE
deleted file mode 100644
index d645695..0000000
--- a/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/NOTICE b/NOTICE
deleted file mode 100644
index 1341f0d..0000000
--- a/NOTICE
+++ /dev/null
@@ -1,5 +0,0 @@
-Apache Streams
-Copyright 2012-2017 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
diff --git a/README.md b/README.md
deleted file mode 100644
index ac71e86..0000000
--- a/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-# Apache Streams
-
-## Overview
-Apache Streams unifies a diverse world of digital profiles and online activities into common formats and vocabularies, and makes these datasets accessible across a variety of databases, devices, and platforms for streaming, browsing, search, sharing, and analytics use-cases.
-
-### What is Streams?
-Apache Streams contains JRE-based modules that developers can use to easily integrate with online data sources and build polyglot indexes of activities, entities, and relationships - all based on public standards such as [Activity Streams](http://activitystrea.ms/), or other published organizational standards.
-
-### Why use Streams?
-Streams contains libraries and patterns for specifying, publishing, and inter-linking schemas, and assists with conversion of activities (posts, shares, likes, follows, etc.) and objects (profiles, pages, photos, videos, etc.) between the representation, format, and encoding preferred by supported data providers (Twitter, Instagram, etc.), and storage services (Cassandra, Elasticsearch, HBase, HDFS, Neo4J, etc.)
-
-### Why is Streams important?
-The project aims to provide simple two-way data interchange with all popular REST APIs in activity streams formats using a universal protocol. No other active open-source project has this ambitious goal, as well as production-worthy implementations for >10 services. Streams compatibility with multiple storage back-ends and ability to be embedded within any java-based real-time or batch data processing platform ensures that its interoperability features come with little technical baggage.
-
-## License
-Copyright (c) 2017. Apache Streams is licensed under the [Apache License 2.0](LICENSE).
diff --git a/pom.xml b/pom.xml
deleted file mode 100644
index af56a56..0000000
--- a/pom.xml
+++ /dev/null
@@ -1,1476 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache</groupId>
- <artifactId>apache</artifactId>
- <version>18</version>
- </parent>
-
- <groupId>org.apache.streams</groupId>
- <artifactId>apache-streams</artifactId>
- <version>0.5.1-SNAPSHOT</version>
-
- <name>Apache Streams</name>
- <description>Apache Streams</description>
-
- <url>http://streams.apache.org/site/${project.version}/streams-project</url>
-
- <licenses>
- <license>
- <name>Apache License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- <distribution>repo</distribution>
- <comments>A business-friendly OSS license</comments>
- </license>
- </licenses>
-
- <scm>
- <connection>scm:git:https://github.com:apache/streams</connection>
- <developerConnection>scm:git:git@github.com:apache/streams.git</developerConnection>
- <url>scm:git:git@github.com:apache/streams.git</url>
- <tag>HEAD</tag>
- </scm>
-
- <issueManagement>
- <system>JIRA</system>
- <url>https://issues.apache.org/jira/browse/STREAMS</url>
- </issueManagement>
-
- <ciManagement>
- <system>jenkins</system>
- <url>http://builds.apache.org/job/streams-project/</url>
- </ciManagement>
-
- <prerequisites>
- <maven>3.0.5</maven>
- </prerequisites>
-
- <mailingLists>
- <mailingList>
- <name>Dev Mailing List</name>
- <post>dev@streams.apache.org</post>
- <subscribe>dev-subscribe@streams.apache.org</subscribe>
- <unsubscribe>dev-unsubscribe@streams.apache.org</unsubscribe>
- <archive>http://mail-archives.apache.org/mod_mbox/incubator-streams-dev/</archive>
- </mailingList>
- <mailingList>
- <name>Commits Mailing List</name>
- <post>commits@streams.apache.org</post>
- <subscribe>commits-subscribe@streams.apache.org</subscribe>
- <unsubscribe>commits-unsubscribe@streams.apache.org</unsubscribe>
- <archive>http://mail-archives.apache.org/mod_mbox/incubator-streams-commits/</archive>
- </mailingList>
- </mailingLists>
-
- <developers>
- <developer>
- <id>rbaxter85</id>
- <name>Ryan Baxter</name>
- <email>rbaxter85@apache.org</email>
- <url>http://twitter.com/ryanjbaxter</url>
- <roles>
- <role>Committer</role>
- </roles>
- </developer>
- <developer>
- <id>sblackmon</id>
- <name>Steve Blackmon</name>
- <email>sblackmon@apache.org</email>
- <url>http://twitter.com/steveblackmon</url>
- <roles>
- <role>PPMC Member</role>
- <role>Committer</role>
- </roles>
- <properties>
- <picUrl>https://avatars.githubusercontent.com/steveblackmon</picUrl>
- </properties>
- </developer>
- <developer>
- <id>rdouglas</id>
- <name>Robert Douglas</name>
- <email>rdouglas@apache.org</email>
- <roles>
- <role>PPMC Member</role>
- <role>Committer</role>
- </roles>
- </developer>
- <developer>
- <id>ate</id>
- <name>Ate Douma</name>
- <email>ate@apache.org</email>
- <url>http://twitter.com/atedouma</url>
- <roles>
- <role>Mentor</role>
- <role>PPMC Member</role>
- <role>Committer</role>
- </roles>
- </developer>
- <developer>
- <id>rebanks</id>
- <name>Ryan Ebanks</name>
- <email>rebanks@apache.org</email>
- <url>http://twitter.com/ebanksre</url>
- <roles>
- <role>PPMC Member</role>
- <role>Committer</role>
- </roles>
- </developer>
- <developer>
- <id>chansen</id>
- <name>Carol Hansen</name>
- <email>chansen@apache.org</email>
- <url>http://twitter.com/carolbirgitta</url>
- <roles>
- <role>PPMC Member</role>
- <role>Committer</role>
- </roles>
- </developer>
- <developer>
- <id>ahart</id>
- <name>Andrew Hart</name>
- <email>ahart@apache.org</email>
- <url>http://twitter.com/andrewfhart</url>
- <roles>
- <role>Mentor</role>
- <role>PPMC Member</role>
- <role>Committer</role>
- </roles>
- </developer>
- <developer>
- <id>mfranklin</id>
- <name>Matt Franklin</name>
- <email>mfranklin@apache.org</email>
- <url>http://twitter.com/mattfranklin</url>
- <roles>
- <role>Mentor</role>
- <role>PPMC Member</role>
- <role>Committer</role>
- </roles>
- </developer>
- <developer>
- <id>letourneau</id>
- <name>Jason Letourneau</name>
- <email>letourneau@apache.org</email>
- <url>http://twitter.com/jletourneau8080</url>
- <roles>
- <role>PPMC Member</role>
- <role>Committer</role>
- </roles>
- </developer>
- <developer>
- <id>craigmcc</id>
- <name>Craig McClanahan</name>
- <email>craigmcc@apache.org</email>
- <url>http://twitter.com/craigmcclanahan</url>
- <roles>
- <role>Mentor</role>
- <role>PPMC Member</role>
- <role>Committer</role>
- </roles>
- </developer>
- <developer>
- <id>dsullivan</id>
- <name>Danny Sullivan</name>
- <email>dsullivan@apache.org</email>
- <url>http://twitter.com/dbsullivan23</url>
- <roles>
- <role>PPMC Member</role>
- <role>Committer</role>
- </roles>
- </developer>
- </developers>
-
- <repositories>
- <repository>
- <id>central-repo</id>
- <name>Maven Repository</name>
- <!-- HTTPS is unavailable for Maven Central -->
- <url>http://repo.maven.apache.org/maven2</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>clojars-repo</id>
- <name>Clojars Repository</name>
- <url>http://clojars.org/repo</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>cloudera-repo</id>
- <name>Cloudera Repository</name>
- <url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- </repositories>
-
- <distributionManagement>
- <repository>
- <id>streams.releases</id>
- <name>Streams Releases</name>
- <url>${release.repository.url}</url>
- </repository>
- <!-- Snapshots (not-releases) are published to this repository -->
- <snapshotRepository>
- <id>streams.snapshots</id>
- <name>Streams Snapshots</name>
- <url>${snapshot.repository.url}</url>
- </snapshotRepository>
- <site>
- <id>site.streams.project</id>
- <url>scm:svn:https://svn.apache.org/repos/asf/incubator/streams/site/trunk/content/site/${project.version}/streams-project</url>
- </site>
- </distributionManagement>
-
- <properties>
-
- <!-- Build Properties -->
- <java.version>1.8</java.version>
- <maven.version>3.3.9</maven.version>
- <scala.suffix>2.10</scala.suffix>
- <scala.version>2.10.6</scala.version>
- <maven.javadoc.failOnError>false</maven.javadoc.failOnError>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <streams.version>0.5.1-SNAPSHOT</streams.version>
- <github.global.server>github</github.global.server>
-
- <!-- Release Properties -->
- <snapshot.repository.url>
- https://repository.apache.org/content/repositories/snapshots
- </snapshot.repository.url>
- <release.repository.url>
- https://repository.apache.org/service/local/staging/deploy/maven2
- </release.repository.url>
-
- <!-- Plugin and Plugin Dependency Versions -->
- <apache-rat.plugin.version>0.12</apache-rat.plugin.version>
- <build-helper.plugin.version>1.8</build-helper.plugin.version>
- <bundle.plugin.version>1.4.3</bundle.plugin.version>
- <checkstyle.plugin.version>2.17</checkstyle.plugin.version>
- <compiler.plugin.version>3.0</compiler.plugin.version>
- <clean.plugin.version>2.6</clean.plugin.version>
- <dependency.plugin.version>2.10</dependency.plugin.version>
- <depgraph.plugin.version>1.0.3</depgraph.plugin.version>
- <docker.plugin.version>0.15.0</docker.plugin.version>
- <download.plugin.version>1.2.1</download.plugin.version>
- <enforcer.plugin.version>1.4.1</enforcer.plugin.version>
- <failsafe.plugin.version>2.19.1</failsafe.plugin.version>
- <jar.plugin.version>2.6</jar.plugin.version>
- <javadoc.plugin.version>2.10.2</javadoc.plugin.version>
- <pax.plugin.version>1.5</pax.plugin.version>
- <remote-resources.plugin.version>1.4</remote-resources.plugin.version>
- <reports.plugin.version>2.9</reports.plugin.version>
- <resources.plugin.version>2.7</resources.plugin.version>
- <scala.plugin.version>3.2.2</scala.plugin.version>
- <scalastyle.plugin.version>0.8.0</scalastyle.plugin.version>
- <scm.plugin.version>1.9.4</scm.plugin.version>
- <shade.plugin.version>2.4.3</shade.plugin.version>
- <site.plugin.version>3.6</site.plugin.version>
- <source.plugin.version>2.4</source.plugin.version>
- <surefire.plugin.version>2.19.1</surefire.plugin.version>
- <wagon.plugin.version>2.5</wagon.plugin.version>
- <war.plugin.version>2.5</war.plugin.version>
-
- <!-- Library Dependency Versions -->
- <jackson.version>2.6.5</jackson.version>
- <jackson-xml-databind.version>0.6.2</jackson-xml-databind.version>
- <aalto.version>1.0.0</aalto.version>
- <joda-time.version>2.9.9</joda-time.version>
- <joda-convert.version>1.8.1</joda-convert.version>
- <json-flattener.version>0.4.1</json-flattener.version>
- <json-schema-validator.version>0.1.10</json-schema-validator.version>
- <juneau.version>6.4.0-incubating</juneau.version>
- <jsonschema2pojo.version>0.4.10</jsonschema2pojo.version>
- <jaxb2.version>0.13.1</jaxb2.version>
- <jaxb2-basics.version>0.11.1</jaxb2-basics.version>
- <jaxbutil.version>1.3.0</jaxbutil.version>
- <junit.version>4.12</junit.version>
- <testng.version>6.11</testng.version>
- <slf4j.version>1.7.25</slf4j.version>
- <log4j.version>1.2.17</log4j.version>
- <logback.version>1.2.3</logback.version>
- <hamcrest.version>1.3</hamcrest.version>
- <commons-codec.version>1.10</commons-codec.version>
- <commons-collections.version>3.2.2</commons-collections.version>
- <commons-io.version>2.5</commons-io.version>
- <commons-lang3.version>3.6</commons-lang3.version>
- <commons-validator.version>1.6</commons-validator.version>
- <typesafe.config.version>1.3.1</typesafe.config.version>
- <reflections.version>0.9.11</reflections.version>
- <guava.version>20.0</guava.version>
- <clojure.version>1.4.0</clojure.version>
- <zookeeper.version>3.4.5</zookeeper.version>
- <netty.version>3.8.0.Final</netty.version>
- <json-path.version>2.4.0</json-path.version>
- <facebook4j.version>2.4.7</facebook4j.version>
- <mockito.version>1.10.19</mockito.version>
- <powermock.version>1.7.1</powermock.version>
- <httpcomponents.core.version>4.4.6</httpcomponents.core.version>
- <httpcomponents.client.version>4.5.3</httpcomponents.client.version>
- <doxia.version>1.7</doxia.version>
-
- <!-- osgi configuration -->
- <org.osgi.service.http.port>8080</org.osgi.service.http.port>
- <org.osgi.service.http.port.secure>8443</org.osgi.service.http.port.secure>
-
- <skipITs>true</skipITs>
- <skipTests>false</skipTests>
-
- <!-- avoid intermittent build failures in streams-plugins -->
- <jvm.options>-Djava.util.Arrays.useLegacyMergeSort=true</jvm.options>
-
- </properties>
-
- <modules>
- <module>streams-core</module>
- <module>streams-config</module>
- <module>streams-contrib</module>
- <module>streams-components</module>
- <module>streams-monitoring</module>
- <module>streams-pojo</module>
- <module>streams-pojo-extensions</module>
- <module>streams-plugins</module>
- <module>streams-runtimes</module>
- <module>streams-schemas</module>
- <module>streams-testing</module>
- <module>streams-util</module>
- <module>streams-verbs</module>
- </modules>
-
- <packaging>pom</packaging>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- </plugin>
- <plugin>
- <artifactId>maven-site-plugin</artifactId>
- </plugin>
- <plugin>
- <artifactId>maven-enforcer-plugin</artifactId>
- </plugin>
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- </plugin>
- <plugin>
- <artifactId>maven-remote-resources-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <artifactId>maven-surefire-plugin</artifactId>
- </plugin>
- <plugin>
- <artifactId>maven-failsafe-plugin</artifactId>
- </plugin>
- </plugins>
- <pluginManagement>
- <plugins>
- <plugin>
- <artifactId>maven-clean-plugin</artifactId>
- <version>${clean.plugin.version}</version>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>${java.version}</source>
- <target>${java.version}</target>
- <encoding>${project.build.sourceEncoding}</encoding>
- <compilerArgs>
- <arg>-J${jvm.options}</arg>
- </compilerArgs>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-remote-resources-plugin</artifactId>
- <version>${remote-resources.plugin.version}</version>
- <executions>
- <execution>
- <goals>
- <goal>process</goal>
- </goals>
- <configuration>
- <resourceBundles combine.children="append">
- <!-- Will generate META-INF/DEPENDENCIES META-INF/LICENSE META-INF/NOTICE -->
- <resourceBundle>org.apache:apache-jar-resource-bundle:1.4</resourceBundle>
- <!-- Will generate META-INF/DISCLAIMER -->
- <resourceBundle>org.apache:apache-incubator-disclaimer-resource-bundle:1.1</resourceBundle>
- </resourceBundles>
- <properties>
- <projectName>${notice.file.project.name}</projectName>
- </properties>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>${resources.plugin.version}</version>
- <configuration>
- <encoding>${project.build.sourceEncoding}</encoding>
- </configuration>
- <executions>
- <execution>
- <id>copy-resources</id>
- <!-- validate is first to run-->
- <!-- copy these files to src/site/resources -->
- <phase>validate</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/src/site/resources</outputDirectory>
- <resources>
- <resource>
- <directory>src/main/jsonschema</directory>
- </resource>
- <resource>
- <directory>src/main/xmlschema</directory>
- </resource>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- <resource>
- <directory>src/site/diagrams</directory>
- </resource>
- <resource>
- <directory>src/test/resources</directory>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.pojo</targetPackage>
- </configuration>
- <executions>
- <execution>
- <phase>generate-sources</phase>
- <goals>
- <goal>generate-sources</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.jvnet.jaxb2.maven2</groupId>
- <artifactId>maven-jaxb2-plugin</artifactId>
- <version>${jaxb2.version}</version>
- <configuration>
- <schemaDirectory>src/main/xmlschema</schemaDirectory>
- <generateDirectory>target/generated-sources/jaxb2</generateDirectory>
- <verbose>true</verbose>
- <debug>true</debug>
- <encoding>${project.build.sourceEncoding}</encoding>
- <forceRegenerate>true</forceRegenerate>
- <removeOldOutput>false</removeOldOutput>
- </configuration>
- <executions>
- <execution>
- <phase>generate-sources</phase>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>${build-helper.plugin.version}</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-enforcer-plugin</artifactId>
- <version>${enforcer.plugin.version}</version>
- <executions>
- <execution>
- <id>enforce-java</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <configuration>
- <rules>
- <requireJavaVersion>
- <message>Java 8 or higher is required to compile this module</message>
- <version>[${java.version},)</version>
- </requireJavaVersion>
- <requireMavenVersion>
- <message>Maven 3.3.9 or higher is required to compile this module</message>
- <version>[${maven.version},)</version>
- </requireMavenVersion>
- </rules>
- </configuration>
- </execution>
- <execution>
- <id>enforce-banned-dependencies</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <configuration>
- <rules>
- <bannedDependencies>
- <excludes>
- <exclude>org.slf4j:slf4j-log4j12</exclude>
- <exclude>org.slf4j:slf4j-jcl</exclude>
- <exclude>org.slf4j:slf4j-jdk14</exclude>
- <exclude>org.log4j:log4j</exclude>
- <exclude>commons-logging:commons-logging</exclude>
- </excludes>
- </bannedDependencies>
- </rules>
- <fail>true</fail>
- </configuration>
- </execution>
- <execution>
- <id>enforce-no-snapshots</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <configuration>
- <rules>
- <requireReleaseDeps>
- <message>No Snapshots Allowed!</message>
- <!-- Don't fail for modules within this project's groupId or sub-groupIds -->
- <excludes>
- <exclude>org.apache.streams:*</exclude>
- <exclude>org.apache.streams.*:*</exclude>
- </excludes>
- <!-- Don't fail for parents being snapshots as the only parent modules are within this project -->
- <failWhenParentIsSnapshot>false</failWhenParentIsSnapshot>
- <onlyWhenRelease>true</onlyWhenRelease>
- </requireReleaseDeps>
- </rules>
- <fail>true</fail>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat.plugin.version}</version>
- <executions>
- <execution>
- <phase>validate</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludes>
- <exclude>.git</exclude>
- <exclude>.gitignore</exclude>
- <exclude>.git/**</exclude>
- <exclude>.idea/**</exclude>
- <exclude>.java-version</exclude>
- <exclude>**/DEPENDENCIES</exclude>
- <exclude>**/LICENSE</exclude>
- <exclude>**/MANIFEST.MF</exclude>
- <exclude>**/NOTICE</exclude>
- <exclude>**/README.*</exclude>
- <exclude>**/target/**</exclude>
- <exclude>**/overlays/**</exclude>
- <exclude>**/src/test/resources/**/*.csv</exclude>
- <exclude>**/src/test/resources/**/*.tsv</exclude>
- <exclude>**/src/site/resources/**/*</exclude>
- <exclude>**/.gitignore</exclude>
- <exclude>**/*.properties</exclude>
- <exclude>**/*.iml</exclude>
- <exclude>**/*.ini</exclude>
- <exclude>**/*.jar</exclude>
- <exclude>**/*.svg</exclude>
- <exclude>dependency-reduced-pom.xml</exclude>
- <exclude>rat.txt</exclude>
- <exclude>apache-maven-3.3.9/**/*</exclude>
- </excludes>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>${surefire.plugin.version}</version>
- <executions>
- <execution>
- <id>test</id>
- <phase>test</phase>
- <goals>
- <goal>test</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <!-- Skips unit tests if the value of skip.unit.tests property is true -->
- <argLine>${jvm.options}</argLine>
- <skipTests>${skipTests}</skipTests>
- <threadCount>1</threadCount>
- <useUnlimitedThreads>false</useUnlimitedThreads>
- <forkCount>1</forkCount>
- <reuseForks>false</reuseForks>
- <!-- Run unit test suites before individuals tests. -->
- <includes>
- <include>**/*Tests.java</include>
- <include>**/*Test.java</include>
- </includes>
- <!-- Excludes integration tests when unit tests are run. -->
- <excludes>
- <exclude>**/*IT.java</exclude>
- <include>**/*ITs.java</include>
- </excludes>
- <reportFormat>plain</reportFormat>
- <properties>
- <property>
- <name>surefire.testng.verbose</name>
- <value>10</value>
- </property>
- </properties>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
- <version>${surefire.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <version>${download.plugin.version}</version>
- <executions>
- <execution>
- <id>download-it-data</id>
- <phase>pre-integration-test</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
- <skip>${skipITs}</skip>
- <url>${testDataBaseURl}/${project.artifactId}.zip</url>
- <unpack>true</unpack>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- <!--<md5>df65b5642f33676313ebe4d5b69a3fff</md5>-->
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <version>${failsafe.plugin.version}</version>
- <executions>
- <execution>
- <id>integration-tests</id>
- <goals>
- <goal>integration-test</goal>
- <goal>verify</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <!-- Skips integration tests if the value of skip.integration.tests property is true -->
- <argLine>${jvm.options}</argLine>
- <skipTests>${skipITs}</skipTests>
- <threadCount>1</threadCount>
- <useUnlimitedThreads>false</useUnlimitedThreads>
- <forkCount>1</forkCount>
- <reuseForks>true</reuseForks>
- <shutdown>exit</shutdown>
- <!-- Excludes unit tests when integration tests are run. -->
- <excludes>
- <exclude>**/*Test.java</exclude>
- <exclude>**/*Tests.java</exclude>
- </excludes>
- <!-- Run integration test suites before individuals tests. -->
- <includes>
- <include>**/*ITs.java</include>
- <include>**/*IT.java</include>
- </includes>
- <reportFormat>plain</reportFormat>
- <properties>
- <property>
- <name>junit</name>
- <value>false</value>
- </property>
- <property>
- <name>surefire.testng.verbose</name>
- <value>10</value>
- </property>
- </properties>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-testng</artifactId>
- <version>${surefire.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <version>${jar.plugin.version}</version>
- <configuration>
- <skipIfEmpty>true</skipIfEmpty>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.ops4j</groupId>
- <artifactId>maven-pax-plugin</artifactId>
- <version>${pax.plugin.version}</version>
- </plugin>
- <plugin>
- <groupId>org.apache.felix</groupId>
- <artifactId>maven-bundle-plugin</artifactId>
- <version>${bundle.plugin.version}</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-site-plugin</artifactId>
- <version>${site.plugin.version}</version>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-module-markdown</artifactId>
- <version>${doxia.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-core</artifactId>
- <version>${doxia.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-module-xhtml</artifactId>
- <version>${doxia.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.wagon</groupId>
- <artifactId>wagon-ssh</artifactId>
- <version>${wagon.plugin.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.wagon</groupId>
- <artifactId>wagon-scm</artifactId>
- <version>${wagon.plugin.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.scm</groupId>
- <artifactId>maven-scm-api</artifactId>
- <version>${scm.plugin.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.scm</groupId>
- <artifactId>maven-scm-provider-svn-commons</artifactId>
- <version>${scm.plugin.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.scm</groupId>
- <artifactId>maven-scm-provider-svnexe</artifactId>
- <version>${scm.plugin.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- <version>${source.plugin.version}</version>
- <executions>
- <execution>
- <id>attach-sources</id>
- <goals>
- <goal>jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <!-- Apache POM doesn't define default war-plugin version, so do it here -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-war-plugin</artifactId>
- <version>${war.plugin.version}</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>${dependency.plugin.version}</version>
- </plugin>
- <plugin>
- <groupId>com.github.ferstl</groupId>
- <artifactId>depgraph-maven-plugin</artifactId>
- <version>${depgraph.plugin.version}</version>
- </plugin>
- <plugin>
- <groupId>io.fabric8</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- <version>${docker.plugin.version}</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <version>${checkstyle.plugin.version}</version>
- <dependencies>
- <dependency>
- <groupId>com.puppycrawl.tools</groupId>
- <artifactId>checkstyle</artifactId>
- <version>7.2</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>validate</id>
- <phase>validate</phase>
- <configuration>
- <configLocation>http://streams.apache.org/site/latest/streams-project/streams-java-checkstyle.xml</configLocation>
- <encoding>UTF-8</encoding>
- <consoleOutput>true</consoleOutput>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <testSourceDirectory>${project.basedir}/src/test/java</testSourceDirectory>
- <failsOnError>false</failsOnError>
- </configuration>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <version>${scalastyle.plugin.version}</version>
- <executions>
- <execution>
- <id>validate</id>
- <phase>validate</phase>
- <configuration>
- <verbose>false</verbose>
- <failOnViolation>false</failOnViolation>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <failOnWarning>false</failOnWarning>
- <sourceDirectory>${project.basedir}/src/main/scala</sourceDirectory>
- <testSourceDirectory>${project.basedir}/src/test/scala</testSourceDirectory>
- <!--<configLocation>https://raw.githubusercontent.com/databricks/sbt-databricks/master/scalastyle-config.xml</configLocation>-->
- <outputFile>${project.build.directory}/scalastyle-output.xml</outputFile>
- <outputEncoding>UTF-8</outputEncoding>
- </configuration>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <version>${scala.plugin.version}</version>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- </dependencies>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>${commons-io.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- <version>${commons-codec.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- <version>${commons-collections.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-validator</groupId>
- <artifactId>commons-validator</artifactId>
- <version>${commons-validator.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>${commons-lang3.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- <version>${httpcomponents.core.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>${httpcomponents.client.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.juneau</groupId>
- <artifactId>juneau-marshall</artifactId>
- <version>${juneau.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.juneau</groupId>
- <artifactId>juneau-marshall-rdf</artifactId>
- <version>${juneau.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.juneau</groupId>
- <artifactId>juneau-rest-client</artifactId>
- <version>${juneau.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.juneau</groupId>
- <artifactId>juneau-rest-server</artifactId>
- <version>${juneau.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.juneau</groupId>
- <artifactId>juneau-svl</artifactId>
- <version>${juneau.version}</version>
- </dependency>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- <version>${typesafe.config.version}</version>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- <version>${reflections.version}</version>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-all</artifactId>
- <version>${hamcrest.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.networknt</groupId>
- <artifactId>json-schema-validator</artifactId>
- <version>${json-schema-validator.version}</version>
- <type>jar</type>
- </dependency>
- <dependency>
- <groupId>com.github.wnameless</groupId>
- <artifactId>json-flattener</artifactId>
- <version>${json-flattener.version}</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>${junit.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- <version>${testng.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock</artifactId>
- <version>${powermock.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>${powermock.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito</artifactId>
- <version>${powermock.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>log4j-over-slf4j</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
- </dependency>
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-core</artifactId>
- <version>${logback.version}</version>
- </dependency>
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- <version>${logback.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- <version>${joda-time.version}</version>
- </dependency>
- <dependency>
- <groupId>org.joda</groupId>
- <artifactId>joda-convert</artifactId>
- <version>${joda-convert.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-joda</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_${scala.suffix}</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-xml</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml</groupId>
- <artifactId>jackson-xml-databind</artifactId>
- <version>${jackson-xml-databind.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-yaml</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml</groupId>
- <artifactId>aalto-xml</artifactId>
- <version>${aalto.version}</version>
- </dependency>
- <dependency>
- <groupId>nz.net.ultraq.jaxb</groupId>
- <artifactId>jaxb-utilities</artifactId>
- <version>${jaxbutil.version}</version>
- </dependency>
- <!-- JsonPath -->
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path</artifactId>
- <version>${json-path.version}</version>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path-assert</artifactId>
- <version>${json-path.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- <version>${zookeeper.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <version>${checkstyle.plugin.version}</version>
- <reportSets>
- <reportSet>
- <reports>
- <report>checkstyle</report>
- <report>checkstyle-aggregate</report>
- </reports>
- </reportSet>
- </reportSets>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <version>${reports.plugin.version}</version>
- <configuration>
- <customBundle>${project.basedir}/src/site/custom/project-info-report.properties</customBundle>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- <skipEmptyReport>false</skipEmptyReport>
- </configuration>
- <reportSets>
- <reportSet>
- <inherited>false</inherited>
- <reports>
- <report>index</report>
- <report>modules</report>
- <report>license</report>
- <report>mailing-list</report>
- <report>project-team</report>
- <report>scm</report>
- <report>issue-tracking</report>
- <report>dependencies</report>
- <report>dependency-info</report>
- <report>dependency-management</report>
- <report>distribution-management</report>
- <report>plugin-management</report>
- </reports>
- </reportSet>
- </reportSets>
- </plugin>
- <plugin>
- <artifactId>maven-site-plugin</artifactId>
- <version>${site.plugin.version}</version>
- <configuration>
- <generateSitemap>true</generateSitemap>
- </configuration>
- </plugin>
- <plugin>
- <groupId>com.github.ferstl</groupId>
- <artifactId>depgraph-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>${javadoc.plugin.version}</version>
- <configuration>
- <additionalparam>-Xdoclint:none</additionalparam>
- <aggregate>false</aggregate>
- <failOnError>false</failOnError>
- <minmemory>128m</minmemory>
- <maxmemory>1g</maxmemory>
- <docfilessubdirs />
- </configuration>
- <reportSets>
- <reportSet>
- <reports>
- <report>javadoc-no-fork</report>
- <report>test-javadoc-no-fork</report>
- </reports>
- </reportSet>
- </reportSets>
- </plugin>
- <plugin>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- </plugin>
- </plugins>
- </reporting>
-
- <profiles>
- <profile>
- <id>quick</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <properties>
- <maven.test.skip.exec>true</maven.test.skip.exec>
- <skipITs>true</skipITs>
- </properties>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
- <id>apache-release</id>
- <properties>
- <skipTests>false</skipTests>
- <maven.test.skip>false</maven.test.skip>
- </properties>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- <version>${source.plugin.version}</version>
- <configuration>
- <skipSource>true</skipSource>
- </configuration>
- <executions>
- <execution>
- <id>attach-sources</id>
- <goals>
- <goal>jar-no-fork</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <skip>false</skip>
- </configuration>
- <executions>
- <execution>
- <id>test-jar</id>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.apache.resources</groupId>
- <artifactId>apache-source-release-assembly-descriptor</artifactId>
- <version>1.0.5</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <id>source-release-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <runOnlyAtExecutionRoot>true</runOnlyAtExecutionRoot>
- <descriptorRefs>
- <descriptorRef>source-release</descriptorRef>
- </descriptorRefs>
- <tarLongFileMode>gnu</tarLongFileMode>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-release-plugin</artifactId>
- <configuration>
- <releaseProfiles>apache-release</releaseProfiles>
- <preparationGoals>clean install</preparationGoals>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <skipTests>false</skipTests>
- <skipExec>true</skipExec>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- </plugin>
- <plugin>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <configuration>
- <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <reportSets>
- <reportSet>
- <id>aggregate</id>
- <reports>
- <report>javadoc-no-fork</report>
- <report>test-javadoc-no-fork</report>
- <report>aggregate</report>
- </reports>
- </reportSet>
- <reportSet>
- <id>html</id>
- <reports>
- <report>javadoc</report>
- </reports>
- </reportSet>
- </reportSets>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-site-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>com.github.ferstl</groupId>
- <artifactId>depgraph-maven-plugin</artifactId>
- <version>${depgraph.plugin.version}</version>
- </plugin>
- </plugins>
- </reporting>
- </profile>
- <profile>
- <id>contrib-check</id>
- <properties>
- <skipTests>false</skipTests>
- <maven.test.skip>false</maven.test.skip>
- </properties>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-</project>
diff --git a/publish-rc.sh b/publish-rc.sh
deleted file mode 100755
index 2e756a2..0000000
--- a/publish-rc.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-REL=$1
-REP=$2
-
-if [[ -z "$REL" || -z "$REP" ]]; then
- echo "You must specify a release version and a staging repo name"
- exit 1
-fi
-
-
-mkdir -p dist
-cd dist
-svn co https://dist.apache.org/repos/dist/dev/incubator/streams dev
-cd dev
-mkdir $REL
-cd $REL
-wget https://repository.apache.org/content/repositories/$REP/org/apache/streams/streams-project/$REL/streams-project-$REL-source-release.zip
-wget https://repository.apache.org/content/repositories/$REP/org/apache/streams/streams-project/$REL/streams-project-$REL-source-release.zip.md5
-wget https://repository.apache.org/content/repositories/$REP/org/apache/streams/streams-project/$REL/streams-project-$REL-source-release.zip.sha1
-wget https://repository.apache.org/content/repositories/$REP/org/apache/streams/streams-project/$REL/streams-project-$REL-source-release.zip.asc
-
-wget https://repository.apache.org/content/repositories/$REP/org/apache/streams/streams-examples/$REL/streams-examples-$REL-source-release.zip
-wget https://repository.apache.org/content/repositories/$REP/org/apache/streams/streams-examples/$REL/streams-examples-$REL-source-release.zip.md5
-wget https://repository.apache.org/content/repositories/$REP/org/apache/streams/streams-examples/$REL/streams-examples-$REL-source-release.zip.sha1
-wget https://repository.apache.org/content/repositories/$REP/org/apache/streams/streams-examples/$REL/streams-examples-$REL-source-release.zip.asc
-
-svn add .*
-svn commit -m "Publishing $REL RC for VOTE"
-cd ../..
\ No newline at end of file
diff --git a/release.sh b/release.sh
deleted file mode 100755
index b461184..0000000
--- a/release.sh
+++ /dev/null
@@ -1,137 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-REL=$1
-DEV=$2
-REPO="-Dmaven.repo.local=/tmp/streams_release"
-
-if [[ -z "$REL" || -z "$DEV" ]]; then
- echo "You must specify a release and new dev version"
- exit 1
-fi
-
-
-mkdir -p /tmp/streams_release
-mkdir -p logs
-
-printInfo() {
- echo "\n"
- mvn -v
- echo "\n"
- docker -v
- echo "\n"
- docker info
- echo "\n\n"
- git status
- git log | head
-}
-
-checkStatus() {
- local output=$1
- if [[ -z "$(tail $output | egrep 'BUILD SUCCESS')" ]]; then
- echo "Release failed"
- exit 1
- fi
-}
-
-#incubator-streams
-git clone https://git-wip-us.apache.org/repos/asf/incubator-streams.git ./incubator-streams-$REL
-cd incubator-streams-$REL
-
-printInfo
-
-mvn clean verify $REPO > ../logs/incubator-streams_unittests.txt
-checkStatus ../logs/incubator-streams_unittests.txt
-
-mvn $REPO clean apache-rat:check -e -DskipTests=true > ../logs/incubator-streams-project_apache-rat_check.txt
-checkStatus ../logs/incubator-streams-project_apache-rat_check.txt
-
-cp ../streams-c84fa47bd759.p12 .
-cp ../application.conf .
-sed -i '' "s#<WORK>#$(pwd)#g" application.conf
-cat application.conf
-
-mvn -PdockerITs $REPO docker:start > ../logs/incubator-streams_docker-start.txt
-checkStatus ../logs/incubator-streams_docker-start.txt
-
-sleep 30
-docker ps
-head *.properties
-
-mvn clean verify $REPO -DskipTests=true -DskipITs=false -DargLine="-Dconfig.file=`pwd`/application.conf" > ../logs/incubator-streams_integrationtests.txt
-checkStatus ../logs/incubator-streams_integrationtests.txt
-
-mvn -PdockerITs $REPO docker:stop > ../logs/incubator-streams_docker-stop.txt
-checkStatus ../logs/incubator-streams_docker-stop.txt
-
-mvn -Papache-release $REPO release:prepare -DpushChanges=false -DautoVersionSubmodules=true -DreleaseVersion=$REL -DdevelopmentVersion=$DEV-SNAPSHOT -Dtag=streams-project-$REL > ../logs/incubator-streams-project_release-prepare.txt
-checkStatus ../logs/incubator-streams-project_release-prepare.txt
-
-mvn -Papache-release $REPO clean install release:perform -Darguments='-Dmaven.test.skip.exec=true' -Dgoals=deploy -DlocalRepoDirectory=. -DlocalCheckout=true > ../logs/incubator-streams-project_release-perform.txt
-checkStatus ../logs/incubator-streams-project_release-perform.txt
-
-cd ..
-
-#streams-examples
-git clone https://git-wip-us.apache.org/repos/asf/incubator-streams-examples.git ./incubator-streams-examples-$REL
-cd incubator-streams-examples-$REL
-
-printInfo
-
-mvn $REPO clean apache-rat:check -e -DskipTests > ../logs/incubator-streams-examples_apache-rat_check.txt
-checkStatus ../logs/incubator-streams-examples_apache-rat_check.txt
-
-mvn $REPO clean verify > ../logs/incubator-streams-examples_unittests.txt
-checkStatus ../logs/incubator-streams-examples_unittests.txt
-
-cp ../streams-c84fa47bd759.p12 .
-cp ../application.conf .
-sed -i '' "s#<WORK>#$(pwd)#g" application.conf
-
-mvn $REPO -PdockerITs -N docker:start > ../logs/incubator-streams-examples_docker-start.txt
-checkStatus ../logs/incubator-streams-examples_docker-start.txt
-
-sleep 30
-docker ps
-head *.properties
-
-mvn $REPO clean verify -DskipTests=true -DskipITs=false -DargLine="-Dconfig.file=`pwd`/application.conf" > ../logs/incubator-streams-examples_integrationtests.txt
-checkStatus ../logs/incubator-streams-examples_integrationtests.txt
-
-mvn $REPO -Papache-release release:prepare -DpushChanges=false -DautoVersionSubmodules=true -DreleaseVersion=$REL -DdevelopmentVersion=$DEV-SNAPSHOT -Dtag=streams-examples-$REL > ../logs/incubator-streams-examples_release-prepare.txt
-checkStatus ../logs/incubator-streams-examples_release-prepare.txt
-
-mvn $REPO -Papache-release clean install release:perform -Darguments='-Dmaven.test.skip.exec=true' -Dgoals=deploy -DlocalRepoDirectory=. -DlocalCheckout=true > ../logs/incubator-streams-examples_release-perform.txt
-checkStatus ../logs/incubator-streams-examples_release-perform.txt
-
-git push origin master
-git push origin streams-examples-$REL
-
-cd ../incubator-streams-$REL
-git push origin master
-git push origin streams-project-$REL
-
-cd..
-
-cat << EOM
-##################################################################
-
- RELEASE COMPLETE
-
-##################################################################
-EOM
\ No newline at end of file
diff --git a/src/main/resources/architecture.dot b/src/main/resources/architecture.dot
deleted file mode 100644
index ea3fd79..0000000
--- a/src/main/resources/architecture.dot
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
- digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- subgraph cluster_upstream {
- label="Upstream Systems";
- upstream_databases [label="Databases", shape="cylinder"]
- upstream_filesystems [label="Filesystems", shape="tab"]
- upstream_generators [label="Generators"]
- upstream_queues [label="Queues", shape="trapezium"]
- }
-
- subgraph cluster_streams {
- label="Apache Streams Pipelines";
- providers [label="Providers"]
- persistReaders [label="PersistReaders"]
- processors [label="Processors"]
- persistWriters [label="PersistWriters"]
- }
-
- subgraph cluster_downstream {
- label="Downstream Systems";
- downstream_databases [label="Databases", shape="cylinder"]
- downstream_filesystems [label="Filesystems", shape="tab"]
- downstream_queues [label="Queues", shape="trapezium"]
- }
-
- upstream_generators -> providers
- upstream_databases -> persistReaders
- upstream_filesystems -> persistReaders
- upstream_queues -> persistReaders
- providers,persistReaders -> processors
- processors -> processors
- processors -> persistWriters
- persistWriters -> downstream_databases
- persistWriters -> downstream_filesystems
- persistWriters -> downstream_queues
-
-}
diff --git a/src/main/resources/example.dot b/src/main/resources/example.dot
deleted file mode 100644
index 9d0428c..0000000
--- a/src/main/resources/example.dot
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
- digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- subgraph cluster_generators {
- label="generators";
- generators_youtube [label="https://api.youtube.com"]
- generators_instagram [label="https://api.instagram.com"]
- }
-
- subgraph cluster_providers {
- label="providers";
- subgraph cluster_providers_youtube {
- label="youtube";
- providers_youtube_channel [label="YoutubeChannelProvider"]
- providers_youtube_video [label="YoutubeUserActivityProvider"]
- }
- subgraph cluster_providers_instagram{
- label="instagram";
- providers_instagram_userinfo [label="InstagramUserInfoCollector"]
- providers_instagram_media [label="InstagramRecentMediaCollector"]
- }
- }
-
- subgraph cluster_processors {
- label="processors";
- processors_youtube_activity [label="YoutubeTypeConverter"]
- processors_youtube_activityobject [label="YoutubeTypeConverter"]
- processors_instagram_activity [label="InstagramTypeConverter"]
- processors_instagram_activityobject [label="InstagramTypeConverter"]
- }
-
- subgraph cluster_persisters_1 {
- label="persisters";
- persisters_kinesis_writer [label="KinesisPersistWriter"]
- persisters_kinesis_writer [label="KinesisPersistWriter"]
- }
-
- subgraph cluster_queues {
- label="queues";
- kinesis [label="kinesis://", shape="trapezium"]
- }
- subgraph cluster_persisters_2 {
- label="persisters";
- persisters_elasticsearch [label="ElasticsearchPersistWriter"]
- persisters_hdfs [label="WebHdfsPersistWriter"]
- persisters_kinesis_reader [label="KinesisPersistReader"]
- persisters_kinesis_reader [label="KinesisPersistReader"]
- }
-
- subgraph cluster_dbs {
- label="dbs";
- elasticsearch [label="elasticsearch", shape="cylinder"]
- hdfs [label="hdfs", shape="tab"]
- }
-
- generators_youtube -> providers_youtube_channel
- generators_youtube -> providers_youtube_video
- providers_youtube_channel -> processors_youtube_activityobject [label="o.a.s.y.Channel"]
- providers_youtube_video -> processors_youtube_activity [label="o.a.s.y.Video"]
-
- generators_instagram -> providers_instagram_userinfo
- generators_instagram -> providers_instagram_media
- providers_instagram_userinfo -> processors_instagram_activityobject [label="o.a.s.i.UserInfoData"]
- providers_instagram_media -> processors_instagram_activity [label="o.a.s.i.MediaFeedData"]
-
- processors_youtube_activityobject -> persisters_kinesis_writer [label="o.a.s.p.j.Page"]
- processors_youtube_activity -> persisters_kinesis_writer [label="o.a.s.p.j.Video\no.a.s.p.j.Comment"]
- processors_instagram_activityobject -> persisters_kinesis_writer [label="o.a.s.p.j.Page"]
- processors_instagram_activity -> persisters_kinesis_writer [label="o.a.s.p.j.Photo\no.a.s.p.j.Video\no.a.s.p.j.Comment"]
-
- persisters_kinesis_writer -> kinesis -> persisters_kinesis_reader [label="o.a.s.p.j.Activity"]
- persisters_kinesis_writer -> kinesis -> persisters_kinesis_reader [label="o.a.s.p.j.ActivityObject"]
-
- persisters_kinesis_reader -> persisters_elasticsearch
- persisters_kinesis_reader -> persisters_hdfs
-
- persisters_elasticsearch -> elasticsearch [label="es://"]
- persisters_hdfs -> hdfs [label="hdfs://"]
-
-}
diff --git a/src/main/resources/images/streams_logo.jpg b/src/main/resources/images/streams_logo.jpg
deleted file mode 100644
index 799ad58..0000000
Binary files a/src/main/resources/images/streams_logo.jpg and /dev/null differ
diff --git a/src/main/resources/integration.dot b/src/main/resources/integration.dot
deleted file mode 100644
index f862f78..0000000
--- a/src/main/resources/integration.dot
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
- digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = LR;
-
- subgraph cluster_git {
- label="git";
- git_streams_project [shape=box, label="streams", URL="https://github.com/apache/streams"];
- git_streams_examples [shape=box, label="streams-examples", URL="https://github.com/apache/streams-examples"];
- }
-
- subgraph cluster_jenkins {
- label="jenkins";
- rankdir = TB;
- jenkins_streams_trunk [label="streams-project", URL="https://builds.apache.org/job/streams-project"];
- jenkins_streams_trunk_verify [label="streams-project-verify", URL="https://builds.apache.org/job/streams-project-verify"];
- jenkins_streams_project_site [label="streams-project-site", URL="https://builds.apache.org/job/streams-project-site"];
- jenkins_streams_examples [label="streams-examples", URL="https://builds.apache.org/job/streams-examples"];
- jenkins_streams_examples_verify [label="streams-examples-verify", URL="https://builds.apache.org/job/streams-examples-verify"];
- jenkins_streams_examples_site [label="streams-examples-site", URL="https://builds.apache.org/job/streams-examples-site"];
- }
-
- subgraph cluster_maven {
- label="maven";
- maven_streams_project [shape=box, label="streams-project", URL="https://repository.apache.org/content/repositories/snapshots/org/apache/streams/streams-project/"];
- maven_streams_examples [shape=box, label="streams-examples", URL="https://repository.apache.org/content/repositories/snapshots/org/apache/streams/streams-examples/"];
- }
-
- subgraph cluster_site {
- label="site";
- site_streams_project [shape=box, label="streams-project", URL="http://streams.apache.org/site/latest/streams-project/"];
- site_streams_examples [shape=box, label="streams-examples", URL="http://streams.apache.org/site/latest/streams-examples/"];
- }
-
- git_streams_project -> jenkins_streams_trunk [style="dashed"]
- git_streams_project -> jenkins_streams_project_site [style="dashed"]
- git_streams_project -> jenkins_streams_trunk_verify [style="dashed"]
- git_streams_examples -> jenkins_streams_examples [style="dashed"]
- git_streams_examples -> jenkins_streams_examples_verify [style="dashed"]
- git_streams_examples -> jenkins_streams_examples_site [style="dashed"]
-
- jenkins_streams_trunk -> maven_streams_project
- jenkins_streams_examples -> maven_streams_examples
-
- jenkins_streams_project_site -> site_streams_project
- jenkins_streams_examples_site -> site_streams_examples
-
- jenkins_streams_trunk -> jenkins_streams_trunk_verify [style="dotted"]
- jenkins_streams_trunk -> jenkins_streams_project_site [style="dotted"]
- jenkins_streams_trunk -> jenkins_streams_examples [style="dotted"]
- jenkins_streams_examples -> jenkins_streams_examples_verify [style="dotted"]
- jenkins_streams_examples -> jenkins_streams_examples_site [style="dotted"]
-
-}
\ No newline at end of file
diff --git a/src/main/resources/streams-java-checkstyle.xml b/src/main/resources/streams-java-checkstyle.xml
deleted file mode 100644
index 67c0692..0000000
--- a/src/main/resources/streams-java-checkstyle.xml
+++ /dev/null
@@ -1,227 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!DOCTYPE module PUBLIC
- "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
- "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
-
-<!--
- Checkstyle configuration that checks the Google coding conventions from Google Java Style
- that can be found at https://google.github.io/styleguide/javaguide.html.
-
- Checkstyle is very configurable. Be sure to read the documentation at
- http://checkstyle.sf.net (or in your downloaded distribution).
-
- To completely disable a check, just comment it out or delete it from the file.
-
- Authors: Max Vetrenko, Ruslan Diachenko, Roman Ivanov.
- -->
-
-<module name = "Checker">
- <property name="charset" value="UTF-8"/>
-
- <property name="severity" value="warning"/>
-
- <property name="fileExtensions" value="java, properties, xml"/>
- <!-- Checks for whitespace -->
- <!-- See http://checkstyle.sf.net/config_whitespace.html -->
- <module name="FileTabCharacter">
- <property name="eachLine" value="true"/>
- </module>
-
- <module name="TreeWalker">
- <module name="OuterTypeFilename"/>
- <module name="IllegalTokenText">
- <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
- <property name="format" value="\\u00(08|09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
- <property name="message" value="Avoid using corresponding octal or Unicode escape."/>
- </module>
- <module name="AvoidEscapedUnicodeCharacters">
- <property name="allowEscapesForControlCharacters" value="true"/>
- <property name="allowByTailComment" value="true"/>
- <property name="allowNonPrintableEscapes" value="true"/>
- </module>
- <module name="LineLength">
- <property name="max" value="140"/>
- <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
- </module>
- <module name="AvoidStarImport"/>
- <module name="OneTopLevelClass"/>
- <module name="NoLineWrap"/>
- <module name="EmptyBlock">
- <property name="option" value="TEXT"/>
- <property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
- </module>
- <module name="NeedBraces"/>
- <module name="LeftCurly">
- <property name="maxLineLength" value="100"/>
- </module>
- <module name="RightCurly"/>
- <module name="RightCurly">
- <property name="option" value="alone"/>
- <property name="tokens" value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO, STATIC_INIT, INSTANCE_INIT"/>
- </module>
- <module name="WhitespaceAround">
- <property name="allowEmptyConstructors" value="true"/>
- <property name="allowEmptyMethods" value="true"/>
- <property name="allowEmptyTypes" value="true"/>
- <property name="allowEmptyLoops" value="true"/>
- <message key="ws.notFollowed"
- value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
- <message key="ws.notPreceded"
- value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
- </module>
- <module name="OneStatementPerLine"/>
- <module name="MultipleVariableDeclarations"/>
- <module name="ArrayTypeStyle"/>
- <module name="MissingSwitchDefault"/>
- <module name="FallThrough"/>
- <module name="UpperEll"/>
- <module name="ModifierOrder"/>
- <module name="EmptyLineSeparator">
- <property name="allowNoEmptyLineBetweenFields" value="true"/>
- </module>
- <module name="SeparatorWrap">
- <property name="tokens" value="DOT"/>
- <property name="option" value="nl"/>
- </module>
- <module name="SeparatorWrap">
- <property name="tokens" value="COMMA"/>
- <property name="option" value="EOL"/>
- </module>
- <module name="PackageName">
- <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
- <message key="name.invalidPattern"
- value="Package name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="TypeName">
- <message key="name.invalidPattern"
- value="Type name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="MemberName">
- <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
- <message key="name.invalidPattern"
- value="Member name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="ParameterName">
- <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
- <message key="name.invalidPattern"
- value="Parameter name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="CatchParameterName">
- <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
- <message key="name.invalidPattern"
- value="Catch parameter name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="LocalVariableName">
- <property name="tokens" value="VARIABLE_DEF"/>
- <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
- <property name="allowOneCharVarInForLoop" value="true"/>
- <message key="name.invalidPattern"
- value="Local variable name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="ClassTypeParameterName">
- <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
- <message key="name.invalidPattern"
- value="Class type name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="MethodTypeParameterName">
- <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
- <message key="name.invalidPattern"
- value="Method type name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="InterfaceTypeParameterName">
- <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
- <message key="name.invalidPattern"
- value="Interface type name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="NoFinalizer"/>
- <module name="GenericWhitespace">
- <message key="ws.followed"
- value="GenericWhitespace ''{0}'' is followed by whitespace."/>
- <message key="ws.preceded"
- value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
- <message key="ws.illegalFollow"
- value="GenericWhitespace ''{0}'' should followed by whitespace."/>
- <message key="ws.notPreceded"
- value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
- </module>
- <module name="Indentation">
- <property name="basicOffset" value="2"/>
- <property name="braceAdjustment" value="0"/>
- <property name="caseIndent" value="2"/>
- <property name="throwsIndent" value="4"/>
- <property name="lineWrappingIndentation" value="4"/>
- <property name="arrayInitIndent" value="2"/>
- </module>
- <module name="AbbreviationAsWordInName">
- <property name="ignoreFinal" value="false"/>
- <property name="allowedAbbreviationLength" value="1"/>
- </module>
- <module name="OverloadMethodsDeclarationOrder"/>
- <module name="VariableDeclarationUsageDistance"/>
- <module name="CustomImportOrder">
- <property name="sortImportsInGroupAlphabetically" value="true"/>
- <property name="separateLineBetweenGroups" value="true"/>
- <property name="customImportOrderRules" value="SAME_PACKAGE(3)###THIRD_PARTY_PACKAGE###STANDARD_JAVA_PACKAGE###STATIC"/>
- </module>
- <module name="MethodParamPad"/>
- <module name="OperatorWrap">
- <property name="option" value="NL"/>
- <property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR, METHOD_REF "/>
- </module>
- <module name="AnnotationLocation">
- <property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
- </module>
- <module name="AnnotationLocation">
- <property name="tokens" value="VARIABLE_DEF"/>
- <property name="allowSamelineMultipleAnnotations" value="true"/>
- </module>
- <module name="NonEmptyAtclauseDescription"/>
- <module name="JavadocTagContinuationIndentation"/>
- <module name="SummaryJavadoc">
- <property name="forbiddenSummaryFragments" value="^@return the *|^This method returns |^A [{]@code [a-zA-Z0-9]+[}]( is a )"/>
- </module>
- <module name="JavadocParagraph"/>
- <module name="AtclauseOrder">
- <property name="tagOrder" value="@param, @return, @throws, @deprecated"/>
- <property name="target" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF, VARIABLE_DEF"/>
- </module>
- <module name="JavadocMethod">
- <property name="scope" value="public"/>
- <property name="allowMissingParamTags" value="true"/>
- <property name="allowMissingThrowsTags" value="true"/>
- <property name="allowMissingReturnTag" value="true"/>
- <property name="minLineCount" value="2"/>
- <property name="allowedAnnotations" value="Override, Test"/>
- <property name="allowThrowsTagsForSubclasses" value="true"/>
- </module>
- <module name="MethodName">
- <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9_]*$"/>
- <message key="name.invalidPattern"
- value="Method name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="SingleLineJavadoc">
- <property name="ignoreInlineTags" value="false"/>
- </module>
- <module name="EmptyCatchBlock">
- <property name="exceptionVariableName" value="expected"/>
- </module>
- <module name="CommentsIndentation"/>
- </module>
-</module>
diff --git a/src/main/resources/streams-java-intellij.xml b/src/main/resources/streams-java-intellij.xml
deleted file mode 100644
index e673c35..0000000
--- a/src/main/resources/streams-java-intellij.xml
+++ /dev/null
@@ -1,493 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<code_scheme name="GoogleStyle">
- <option name="JAVA_INDENT_OPTIONS">
- <value>
- <option name="INDENT_SIZE" value="2" />
- <option name="CONTINUATION_INDENT_SIZE" value="4" />
- <option name="TAB_SIZE" value="8" />
- <option name="USE_TAB_CHARACTER" value="false" />
- <option name="SMART_TABS" value="false" />
- <option name="LABEL_INDENT_SIZE" value="0" />
- <option name="LABEL_INDENT_ABSOLUTE" value="false" />
- <option name="USE_RELATIVE_INDENTS" value="false" />
- </value>
- </option>
- <option name="CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND" value="99" />
- <option name="NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND" value="99" />
- <option name="IMPORT_LAYOUT_TABLE">
- <value>
- <package name="org.apache.streams" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="android" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="antenna" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="antlr" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="ar" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="asposewobfuscated" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="asquare" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="atg" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="au" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="beaver" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="bibtex" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="bmsi" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="bsh" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="ccl" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="cern" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="ChartDirector" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="checkers" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="com" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="COM" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="common" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="contribs" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="corejava" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="cryptix" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="cybervillains" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="dalvik" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="danbikel" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="de" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="EDU" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="eg" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="eu" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="examples" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="fat" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="fit" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="fitlibrary" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="fmpp" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="freemarker" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="gnu" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="groovy" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="groovyjarjarantlr" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="groovyjarjarasm" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="hak" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="hep" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="ie" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="imageinfo" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="info" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="it" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="jal" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="Jama" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="japa" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="japacheckers" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="jas" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="jasmin" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="javancss" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="javanet" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="javassist" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="javazoom" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="java_cup" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="jcifs" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="jetty" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="JFlex" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="jj2000" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="jline" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="jp" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="JSci" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="jsr166y" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="junit" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="jxl" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="jxxload_help" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="kawa" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="kea" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="libcore" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="libsvm" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="lti" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="memetic" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="mt" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="mx4j" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="net" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="netscape" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="nl" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="nu" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="oauth" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="ognl" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="opennlp" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="oracle" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="org" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="penn2dg" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="pennconverter" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="pl" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="prefuse" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="proguard" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="repackage" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="scm" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="se" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="serp" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="simple" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="soot" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="sqlj" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="src" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="ssa" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="sun" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="sunlabs" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="tcl" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="testdata" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="testshell" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="testsuite" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="twitter4j" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="uk" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="ViolinStrings" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="weka" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="wet" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="winstone" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="woolfel" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="wowza" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="java" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="javax" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="" withSubpackages="true" static="true" />
- </value>
- </option>
- <option name="RIGHT_MARGIN" value="140" />
- <option name="JD_P_AT_EMPTY_LINES" value="false" />
- <option name="JD_KEEP_EMPTY_PARAMETER" value="false" />
- <option name="JD_KEEP_EMPTY_EXCEPTION" value="false" />
- <option name="JD_KEEP_EMPTY_RETURN" value="false" />
- <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
- <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
- <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1" />
- <option name="ALIGN_MULTILINE_PARAMETERS_IN_CALLS" value="true" />
- <option name="ALIGN_MULTILINE_BINARY_OPERATION" value="true" />
- <option name="ALIGN_MULTILINE_ASSIGNMENT" value="true" />
- <option name="ALIGN_MULTILINE_TERNARY_OPERATION" value="true" />
- <option name="ALIGN_MULTILINE_THROWS_LIST" value="true" />
- <option name="ALIGN_MULTILINE_EXTENDS_LIST" value="true" />
- <option name="ALIGN_MULTILINE_PARENTHESIZED_EXPRESSION" value="true" />
- <option name="ALIGN_MULTILINE_ARRAY_INITIALIZER_EXPRESSION" value="true" />
- <option name="CALL_PARAMETERS_WRAP" value="1" />
- <option name="METHOD_PARAMETERS_WRAP" value="1" />
- <option name="EXTENDS_LIST_WRAP" value="1" />
- <option name="THROWS_LIST_WRAP" value="1" />
- <option name="EXTENDS_KEYWORD_WRAP" value="1" />
- <option name="THROWS_KEYWORD_WRAP" value="1" />
- <option name="METHOD_CALL_CHAIN_WRAP" value="1" />
- <option name="BINARY_OPERATION_WRAP" value="1" />
- <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
- <option name="TERNARY_OPERATION_WRAP" value="1" />
- <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
- <option name="FOR_STATEMENT_WRAP" value="1" />
- <option name="ARRAY_INITIALIZER_WRAP" value="1" />
- <option name="ASSIGNMENT_WRAP" value="5" />
- <option name="WRAP_COMMENTS" value="true" />
- <option name="IF_BRACE_FORCE" value="3" />
- <option name="DOWHILE_BRACE_FORCE" value="3" />
- <option name="WHILE_BRACE_FORCE" value="3" />
- <option name="FOR_BRACE_FORCE" value="3" />
- <ADDITIONAL_INDENT_OPTIONS fileType="css">
- <option name="INDENT_SIZE" value="4" />
- <option name="CONTINUATION_INDENT_SIZE" value="8" />
- <option name="TAB_SIZE" value="4" />
- <option name="USE_TAB_CHARACTER" value="false" />
- <option name="SMART_TABS" value="false" />
- <option name="LABEL_INDENT_SIZE" value="0" />
- <option name="LABEL_INDENT_ABSOLUTE" value="false" />
- <option name="USE_RELATIVE_INDENTS" value="false" />
- </ADDITIONAL_INDENT_OPTIONS>
- <ADDITIONAL_INDENT_OPTIONS fileType="haml">
- <option name="INDENT_SIZE" value="2" />
- <option name="CONTINUATION_INDENT_SIZE" value="8" />
- <option name="TAB_SIZE" value="4" />
- <option name="USE_TAB_CHARACTER" value="false" />
- <option name="SMART_TABS" value="false" />
- <option name="LABEL_INDENT_SIZE" value="0" />
- <option name="LABEL_INDENT_ABSOLUTE" value="false" />
- <option name="USE_RELATIVE_INDENTS" value="false" />
- </ADDITIONAL_INDENT_OPTIONS>
- <ADDITIONAL_INDENT_OPTIONS fileType="java">
- <option name="INDENT_SIZE" value="2" />
- <option name="CONTINUATION_INDENT_SIZE" value="4" />
- <option name="TAB_SIZE" value="8" />
- <option name="USE_TAB_CHARACTER" value="false" />
- <option name="SMART_TABS" value="false" />
- <option name="LABEL_INDENT_SIZE" value="0" />
- <option name="LABEL_INDENT_ABSOLUTE" value="false" />
- <option name="USE_RELATIVE_INDENTS" value="false" />
- </ADDITIONAL_INDENT_OPTIONS>
- <ADDITIONAL_INDENT_OPTIONS fileType="js">
- <option name="INDENT_SIZE" value="4" />
- <option name="CONTINUATION_INDENT_SIZE" value="4" />
- <option name="TAB_SIZE" value="4" />
- <option name="USE_TAB_CHARACTER" value="false" />
- <option name="SMART_TABS" value="false" />
- <option name="LABEL_INDENT_SIZE" value="0" />
- <option name="LABEL_INDENT_ABSOLUTE" value="false" />
- <option name="USE_RELATIVE_INDENTS" value="false" />
- </ADDITIONAL_INDENT_OPTIONS>
- <ADDITIONAL_INDENT_OPTIONS fileType="jsp">
- <option name="INDENT_SIZE" value="4" />
- <option name="CONTINUATION_INDENT_SIZE" value="8" />
- <option name="TAB_SIZE" value="4" />
- <option name="USE_TAB_CHARACTER" value="false" />
- <option name="SMART_TABS" value="false" />
- <option name="LABEL_INDENT_SIZE" value="0" />
- <option name="LABEL_INDENT_ABSOLUTE" value="false" />
- <option name="USE_RELATIVE_INDENTS" value="false" />
- </ADDITIONAL_INDENT_OPTIONS>
- <ADDITIONAL_INDENT_OPTIONS fileType="php">
- <option name="INDENT_SIZE" value="4" />
- <option name="CONTINUATION_INDENT_SIZE" value="8" />
- <option name="TAB_SIZE" value="4" />
- <option name="USE_TAB_CHARACTER" value="false" />
- <option name="SMART_TABS" value="false" />
- <option name="LABEL_INDENT_SIZE" value="0" />
- <option name="LABEL_INDENT_ABSOLUTE" value="false" />
- <option name="USE_RELATIVE_INDENTS" value="false" />
- </ADDITIONAL_INDENT_OPTIONS>
- <ADDITIONAL_INDENT_OPTIONS fileType="sass">
- <option name="INDENT_SIZE" value="2" />
- <option name="CONTINUATION_INDENT_SIZE" value="8" />
- <option name="TAB_SIZE" value="4" />
- <option name="USE_TAB_CHARACTER" value="false" />
- <option name="SMART_TABS" value="false" />
- <option name="LABEL_INDENT_SIZE" value="0" />
- <option name="LABEL_INDENT_ABSOLUTE" value="false" />
- <option name="USE_RELATIVE_INDENTS" value="false" />
- </ADDITIONAL_INDENT_OPTIONS>
- <ADDITIONAL_INDENT_OPTIONS fileType="xml">
- <option name="INDENT_SIZE" value="4" />
- <option name="CONTINUATION_INDENT_SIZE" value="8" />
- <option name="TAB_SIZE" value="4" />
- <option name="USE_TAB_CHARACTER" value="false" />
- <option name="SMART_TABS" value="false" />
- <option name="LABEL_INDENT_SIZE" value="0" />
- <option name="LABEL_INDENT_ABSOLUTE" value="false" />
- <option name="USE_RELATIVE_INDENTS" value="false" />
- </ADDITIONAL_INDENT_OPTIONS>
- <ADDITIONAL_INDENT_OPTIONS fileType="yml">
- <option name="INDENT_SIZE" value="2" />
- <option name="CONTINUATION_INDENT_SIZE" value="8" />
- <option name="TAB_SIZE" value="4" />
- <option name="USE_TAB_CHARACTER" value="false" />
- <option name="SMART_TABS" value="false" />
- <option name="LABEL_INDENT_SIZE" value="0" />
- <option name="LABEL_INDENT_ABSOLUTE" value="false" />
- <option name="USE_RELATIVE_INDENTS" value="false" />
- </ADDITIONAL_INDENT_OPTIONS>
- <codeStyleSettings language="ECMA Script Level 4">
- <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
- <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
- <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1" />
- <option name="ALIGN_MULTILINE_PARAMETERS_IN_CALLS" value="true" />
- <option name="ALIGN_MULTILINE_BINARY_OPERATION" value="true" />
- <option name="ALIGN_MULTILINE_ASSIGNMENT" value="true" />
- <option name="ALIGN_MULTILINE_TERNARY_OPERATION" value="true" />
- <option name="ALIGN_MULTILINE_THROWS_LIST" value="true" />
- <option name="ALIGN_MULTILINE_EXTENDS_LIST" value="true" />
- <option name="ALIGN_MULTILINE_PARENTHESIZED_EXPRESSION" value="true" />
- <option name="ALIGN_MULTILINE_ARRAY_INITIALIZER_EXPRESSION" value="true" />
- <option name="CALL_PARAMETERS_WRAP" value="1" />
- <option name="METHOD_PARAMETERS_WRAP" value="1" />
- <option name="EXTENDS_LIST_WRAP" value="1" />
- <option name="THROWS_LIST_WRAP" value="1" />
- <option name="EXTENDS_KEYWORD_WRAP" value="1" />
- <option name="THROWS_KEYWORD_WRAP" value="1" />
- <option name="METHOD_CALL_CHAIN_WRAP" value="1" />
- <option name="BINARY_OPERATION_WRAP" value="1" />
- <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
- <option name="TERNARY_OPERATION_WRAP" value="1" />
- <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
- <option name="FOR_STATEMENT_WRAP" value="1" />
- <option name="ARRAY_INITIALIZER_WRAP" value="1" />
- <option name="ASSIGNMENT_WRAP" value="5" />
- <option name="WRAP_COMMENTS" value="true" />
- <option name="IF_BRACE_FORCE" value="3" />
- <option name="DOWHILE_BRACE_FORCE" value="3" />
- <option name="WHILE_BRACE_FORCE" value="3" />
- <option name="FOR_BRACE_FORCE" value="3" />
- <option name="PARENT_SETTINGS_INSTALLED" value="true" />
- </codeStyleSettings>
- <codeStyleSettings language="JavaScript">
- <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
- <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
- <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1" />
- <option name="ALIGN_MULTILINE_PARAMETERS_IN_CALLS" value="true" />
- <option name="ALIGN_MULTILINE_BINARY_OPERATION" value="true" />
- <option name="ALIGN_MULTILINE_ASSIGNMENT" value="true" />
- <option name="ALIGN_MULTILINE_TERNARY_OPERATION" value="true" />
- <option name="ALIGN_MULTILINE_THROWS_LIST" value="true" />
- <option name="ALIGN_MULTILINE_EXTENDS_LIST" value="true" />
- <option name="ALIGN_MULTILINE_PARENTHESIZED_EXPRESSION" value="true" />
- <option name="ALIGN_MULTILINE_ARRAY_INITIALIZER_EXPRESSION" value="true" />
- <option name="CALL_PARAMETERS_WRAP" value="1" />
- <option name="METHOD_PARAMETERS_WRAP" value="1" />
- <option name="EXTENDS_LIST_WRAP" value="1" />
- <option name="THROWS_LIST_WRAP" value="1" />
- <option name="EXTENDS_KEYWORD_WRAP" value="1" />
- <option name="THROWS_KEYWORD_WRAP" value="1" />
- <option name="METHOD_CALL_CHAIN_WRAP" value="1" />
- <option name="BINARY_OPERATION_WRAP" value="1" />
- <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
- <option name="TERNARY_OPERATION_WRAP" value="1" />
- <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
- <option name="FOR_STATEMENT_WRAP" value="1" />
- <option name="ARRAY_INITIALIZER_WRAP" value="1" />
- <option name="ASSIGNMENT_WRAP" value="5" />
- <option name="WRAP_COMMENTS" value="true" />
- <option name="IF_BRACE_FORCE" value="3" />
- <option name="DOWHILE_BRACE_FORCE" value="3" />
- <option name="WHILE_BRACE_FORCE" value="3" />
- <option name="FOR_BRACE_FORCE" value="3" />
- <option name="PARENT_SETTINGS_INSTALLED" value="true" />
- </codeStyleSettings>
- <codeStyleSettings language="PHP">
- <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
- <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
- <option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1" />
- <option name="ALIGN_MULTILINE_ASSIGNMENT" value="true" />
- <option name="ALIGN_MULTILINE_TERNARY_OPERATION" value="true" />
- <option name="ALIGN_MULTILINE_THROWS_LIST" value="true" />
- <option name="ALIGN_MULTILINE_EXTENDS_LIST" value="true" />
- <option name="ALIGN_MULTILINE_PARENTHESIZED_EXPRESSION" value="true" />
- <option name="CALL_PARAMETERS_WRAP" value="1" />
- <option name="METHOD_PARAMETERS_WRAP" value="1" />
- <option name="EXTENDS_LIST_WRAP" value="1" />
- <option name="THROWS_LIST_WRAP" value="1" />
- <option name="EXTENDS_KEYWORD_WRAP" value="1" />
- <option name="THROWS_KEYWORD_WRAP" value="1" />
- <option name="METHOD_CALL_CHAIN_WRAP" value="1" />
- <option name="BINARY_OPERATION_WRAP" value="1" />
- <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
- <option name="TERNARY_OPERATION_WRAP" value="1" />
- <option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="true" />
- <option name="FOR_STATEMENT_WRAP" value="1" />
- <option name="ARRAY_INITIALIZER_WRAP" value="1" />
- <option name="ASSIGNMENT_WRAP" value="5" />
- <option name="WRAP_COMMENTS" value="true" />
- <option name="IF_BRACE_FORCE" value="3" />
- <option name="DOWHILE_BRACE_FORCE" value="3" />
- <option name="WHILE_BRACE_FORCE" value="3" />
- <option name="FOR_BRACE_FORCE" value="3" />
- <option name="PARENT_SETTINGS_INSTALLED" value="true" />
- </codeStyleSettings>
-</code_scheme>
-
diff --git a/src/site/custom/project-info-report.properties b/src/site/custom/project-info-report.properties
deleted file mode 100644
index 49033c8..0000000
--- a/src/site/custom/project-info-report.properties
+++ /dev/null
@@ -1,299 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-report.ci-management.jenkins.intro = This project uses {Jenkins, http://jenkins-ci.org/}.
-report.ci-management.name = CI Management
-report.ci-management.nocim = No continuous integration management system is defined. Please check back at a later date.
-report.ci-management.notifiers.column.address = Address
-report.ci-management.notifiers.column.configuration = Configuration
-report.ci-management.notifiers.column.type = Type
-report.ci-management.notifiers.intro = Configuration for notifying developers/users when a build is unsuccessful, including user information and notification mode.
-report.ci-management.notifiers.nolist = No notifiers are defined. Please check back at a later date.
-report.ci-management.notifiers.title = Notifiers
-report.ci-management.nourl = No url to the continuous integration system is defined.
-report.ci-management.overview.title = Overview
-report.ci-management.title = CI Management Yo
-report.ci-management.url = The following is a link to the continuous integration system used by the project, yo:
-report.dependencies.column.artifactId = ArtifactId
-report.dependencies.column.classifier = Classifier
-report.dependencies.column.description = Description
-report.dependencies.column.groupId = GroupId
-report.dependencies.column.licenses = Licenses
-report.dependencies.column.optional = Optional
-report.dependencies.column.isOptional = Yes
-report.dependencies.column.isNotOptional = No
-report.dependencies.column.type = Type
-report.dependencies.column.url = URL
-report.dependencies.column.version = Version
-report.dependencies.description = This document lists the project's dependencies and provides information on each dependency.
-report.dependencies.file.details.cell.debuginformation.yes = Yes
-report.dependencies.file.details.cell.debuginformation.no = No
-report.dependencies.file.details.cell.sealed.yes = Yes
-report.dependencies.file.details.cell.sealed.no = No
-report.dependencies.file.details.column.classes = Classes
-report.dependencies.file.details.column.debuginformation = Debug Information
-report.dependencies.file.details.column.entries = Entries
-report.dependencies.file.details.column.file = Filename
-report.dependencies.file.details.column.javaVersion = Java Version
-report.dependencies.file.details.column.packages = Packages
-report.dependencies.file.details.column.sealed = Sealed
-report.dependencies.file.details.column.size = Size
-report.dependencies.file.details.column.size.gb = GB
-report.dependencies.file.details.column.size.mb = MB
-report.dependencies.file.details.column.size.kb = kB
-report.dependencies.file.details.columntitle.debuginformation = Indicates whether these dependencies have been compiled with debug information.
-report.dependencies.file.details.title = Dependency File Details
-report.dependencies.file.details.total = Total
-report.dependencies.graph.icon.close = [Close]
-report.dependencies.graph.icon.information = [Information]
-report.dependencies.graph.tables.licenses = Licenses
-report.dependencies.graph.tables.unknown = Unknown
-report.dependencies.graph.title = Project Dependency Graph
-report.dependencies.graph.tree.title = Dependency Tree
-report.dependencies.intro.compile = The following is a list of compile dependencies for this project. These dependencies are required to compile and run the application:
-report.dependencies.intro.provided = The following is a list of provided dependencies for this project. These dependencies are required to compile the application, but should be provided by default when using the library:
-report.dependencies.intro.runtime = The following is a list of runtime dependencies for this project. These dependencies are required to run the application:
-report.dependencies.intro.system = The following is a list of system dependencies for this project. These dependencies are required to compile the application:
-report.dependencies.intro.test = The following is a list of test dependencies for this project. These dependencies are only required to compile and run unit tests for the application:
-report.dependencies.name = Dependencies
-report.dependencies.nolist = There are no dependencies for this project. It is a standalone application that does not depend on any other project.
-report.dependencies.repo.locations.artifact.breakdown = Repository locations for each of the Dependencies.
-report.dependencies.repo.locations.cell.release.disabled = No
-report.dependencies.repo.locations.cell.release.enabled = Yes
-report.dependencies.repo.locations.cell.snapshot.disabled = No
-report.dependencies.repo.locations.cell.snapshot.enabled = Yes
-report.dependencies.repo.locations.cell.blacklisted.disabled = No
-report.dependencies.repo.locations.cell.blacklisted.enabled = Yes
-report.dependencies.repo.locations.column.artifact = Artifact
-report.dependencies.repo.locations.column.blacklisted = Blacklisted
-report.dependencies.repo.locations.column.release = Release
-report.dependencies.repo.locations.column.repoid = Repo ID
-report.dependencies.repo.locations.column.snapshot = Snapshot
-report.dependencies.repo.locations.column.url = URL
-report.dependencies.repo.locations.title = Dependency Repository Locations
-report.dependencies.title = Project Dependencies
-report.dependencies.unnamed = Unnamed
-report.dependencies.transitive.intro = The following is a list of transitive dependencies for this project. Transitive dependencies are the dependencies of the project dependencies.
-report.dependencies.transitive.nolist = No transitive dependencies are required for this project.
-report.dependencies.transitive.title = Project Transitive Dependencies
-report.dependency-convergence.convergence.caption = Dependencies used in modules
-report.dependency-convergence.convergence.single.caption = Dependencies used in this project
-report.dependency-convergence.description = This document presents the convergence of dependency versions across the entire project, and its sub modules.
-report.dependency-convergence.icon.error = [Error]
-report.dependency-convergence.icon.success = [Success]
-report.dependency-convergence.legend = Legend:
-report.dependency-convergence.legend.different = At least one dependency has a differing version of the dependency or has SNAPSHOT dependencies.
-report.dependency-convergence.legend.shared = All modules/dependencies share one version of the dependency.
-report.dependency-convergence.name = Dependency Convergence
-report.dependency-convergence.reactor.name = Reactor Dependency Convergence
-report.dependency-convergence.reactor.title = Reactor Dependency Convergence
-report.dependency-convergence.stats.artifacts = Number of unique artifacts (NOA):
-report.dependency-convergence.stats.caption = Statistics:
-report.dependency-convergence.stats.convergence = Convergence (NOD/NOA):
-report.dependency-convergence.stats.dependencies = Number of dependencies (NOD):
-report.dependency-convergence.stats.readyrelease = Ready for release (100% convergence and no SNAPSHOTS):
-report.dependency-convergence.stats.readyrelease.error = Error
-report.dependency-convergence.stats.readyrelease.error.convergence = You do not have 100% convergence.
-report.dependency-convergence.stats.readyrelease.error.snapshots = You have SNAPSHOT dependencies.
-report.dependency-convergence.stats.readyrelease.success = Success
-report.dependency-convergence.stats.conflicting = Number of version-conflicting artifacts (NOC):
-report.dependency-convergence.stats.snapshots = Number of SNAPSHOT artifacts (NOS):
-report.dependency-convergence.stats.modules = Number of modules:
-report.dependency-convergence.title = Dependency Convergence
-report.dependency-info.name = Dependency Information
-report.dependency-info.title = Dependency Information
-report.dependency-info.description = This document describes how to to include this project as a dependency using various dependency management tools.
-report.index.nodescription = There is currently no description associated with this project.
-report.index.title = About
-report.issue-management.bugzilla.intro = This project uses {Bugzilla, http://www.bugzilla.org/}.
-report.issue-management.custom.intro = This project uses %issueManagementSystem% to manage its issues.
-report.issue-management.description = This document provides information on the issue management system used in this project.
-report.issue-management.general.intro = This project uses an issue management system to manage its issues.
-report.issue-management.intro = Issues, bugs, and feature requests should be submitted to the following issue management system for this project.
-report.issue-management.jira.intro = This project uses {JIRA, http://www.atlassian.com/software/jira}.
-report.issue-management.name = Issue Management
-report.issue-management.noissueManagement = No issue management system is defined. Please check back at a later date.
-report.issue-management.overview.title = Overview
-report.issue-management.scarab.intro = This project uses {Scarab, http://scarab.tigris.org/}.
-report.issue-management.title = Issue Management
-report.licenses.description = This document lists the project license(s).
-report.licenses.multiple = This project is provided under multiple licenses:
-report.licenses.name = Licenses
-report.licenses.nolicense = No licenses are defined for this project.
-report.licenses.overview.intro = Typically the licenses listed for the project are that of the project itself, and not of dependencies.
-report.licenses.overview.title = Overview
-report.licenses.originalText = [Original text]
-report.licenses.copy = Copy of the license follows:
-report.licenses.title = Project Licenses
-report.licenses.unnamed = Unnamed
-report.mailing-lists.column.archive = Archive
-report.mailing-lists.column.name = Name
-report.mailing-lists.column.otherArchives = Other Archives
-report.mailing-lists.column.post = Post
-report.mailing-lists.column.subscribe = Subscribe
-report.mailing-lists.column.unsubscribe = Unsubscribe
-report.mailing-lists.description = This document provides subscription and archive information for this project's mailing lists.
-report.mailing-lists.intro = These are the mailing lists that have been established for this project. For each list, there is a subscribe, unsubscribe, and an archive link.
-report.mailing-lists.name = Mailing Lists
-report.mailing-lists.nolist = There are no mailing lists currently associated with this project.
-report.mailing-lists.title = Project Mailing Lists
-report.scm.accessbehindfirewall.cvs.intro = For those developers who are stuck behind a corporate firewall, {CVSGrab, http://cvsgrab.sourceforge.net/} can use the viewcvs web interface to checkout the source code.
-report.scm.accessbehindfirewall.general.intro = Refer to the documentation of the SCM used for more information about access behind a firewall.
-report.scm.accessbehindfirewall.svn.intro = For those users who are stuck behind a corporate firewall which is blocking HTTP access to the Subversion repository, you can try to access it via the developer connection:
-report.scm.accessbehindfirewall.title = Access from Behind a Firewall
-report.scm.accessthroughtproxy.svn.intro1 = The Subversion client can go through a proxy, if you configure it to do so. First, edit your "servers" configuration file to indicate which proxy to use. The file's location depends on your operating system. On Linux or Unix it is located in the directory "~/.subversion". On Windows it is in "%APPDATA%\\Subversion". (Try "echo %APPDATA%", note this is a hidden directory.)
-report.scm.accessthroughtproxy.svn.intro2 = There are comments in the file explaining what to do. If you don't have that file, get the latest Subversion client and run any command; this will cause the configuration directory and template files to be created.
-report.scm.accessthroughtproxy.svn.intro3 = Example: Edit the 'servers' file and add something like:
-report.scm.accessthroughtproxy.title = Access Through a Proxy
-report.scm.anonymousaccess.cvs.intro = This project's CVS repository can be checked out through anonymous CVS with the following instruction set. When prompted for a password for anonymous, simply press the Enter key.
-report.scm.anonymousaccess.general.intro = Refer to the documentation of the SCM used for more information about anonymously check out. The connection url is:
-report.scm.anonymousaccess.git.intro = The source can be checked out anonymously from Git with this command (See {http://git-scm.com/docs/git-clone,http://git-scm.com/docs/git-clone}):
-report.scm.anonymousaccess.hg.intro = The source can be checked out anonymously from Mercurial with this command (See {http://www.selenic.com/mercurial/hg.1.html#clone,http://www.selenic.com/mercurial/hg.1.html#clone}):
-report.scm.anonymousaccess.svn.intro = The source can be checked out anonymously from Subversion with this command:
-report.scm.anonymousaccess.title = Anonymous Access
-report.scm.clearcase.intro = This project uses {ClearCase, http://www-306.ibm.com/software/awdtools/clearcase/} to manage its source code. Informations on ClearCase use can be found at {http://www.redbooks.ibm.com/redbooks/pdfs/sg246399.pdf, http://www.redbooks.ibm.com/redbooks/pdfs/sg246399.pdf}.
-report.scm.cvs.intro = This project uses {Concurrent Versions System, http://www.cvshome.org/} to manage its source code. Instructions on CVS use can be found at {http://cvsbook.red-bean.com/, http://cvsbook.red-bean.com/}.
-report.scm.description = This document lists ways to access the online source repository.
-report.scm.devaccess.clearcase.intro = Only project developers can access the ClearCase tree via this method. Substitute username with the proper value.
-report.scm.devaccess.cvs.intro = Only project developers can access the CVS tree via this method. Substitute username with the proper value.
-report.scm.devaccess.general.intro = Refer to the documentation of the SCM used for more information about developer check out. The connection url is:
-report.scm.devaccess.git.intro = Only project developers can access the Git tree via this method (See {http://git-scm.com/docs/git-clone,http://git-scm.com/docs/git-clone}).
-report.scm.devaccess.hg.intro = Only project developers can access the Mercurial tree via this method (See {http://www.selenic.com/mercurial/hg.1.html#clone,http://www.selenic.com/mercurial/hg.1.html#clone}).
-report.scm.devaccess.perforce.intro = Only project developers can access the Perforce tree via this method. Substitute username and password with the proper values.
-report.scm.devaccess.starteam.intro = Only project developers can access the Starteam tree via this method. Substitute username with the proper value.
-report.scm.devaccess.svn.intro1.https = Everyone can access the Subversion repository via HTTP, but committers must checkout the Subversion repository via HTTPS.
-report.scm.devaccess.svn.intro1.other = Committers must checkout the Subversion repository.
-report.scm.devaccess.svn.intro1.svn = Committers must checkout the Subversion repository via SVN.
-report.scm.devaccess.svn.intro1.svnssh = Committers must checkout the Subversion repository via SVN+SSH.
-report.scm.devaccess.svn.intro2 = To commit changes to the repository, execute the following command to commit your changes (svn will prompt you for your password):
-report.scm.devaccess.title = Developer Access
-report.scm.general.intro = This project uses a source code management system to manage its source code.
-report.scm.name = Source Code Management
-report.scm.noscm = No source configuration management system is defined. Please check back at a later date.
-report.scm.overview.title = Overview
-report.scm.git.intro = This project uses {Git, http://git-scm.com/} to manage its source code. Instructions on Git use can be found at {http://git-scm.com/documentation,http://git-scm.com/documentation}.
-report.scm.hg.intro = This project uses {Mercurial, http://mercurial.selenic.com/wiki/} to manage its source code. Instructions on Mercurial use can be found at {http://hgbook.red-bean.com/read/, http://hgbook.red-bean.com/read/}.
-report.scm.perforce.intro = This project uses {Perforce, http://www.perforce.com/} to manage its source code. Instructions on Perforce use can be found at {http://www.perforce.com/perforce/doc.051/manuals/cmdref/index.html, http://www.perforce.com/perforce/doc.051/manuals/cmdref/index.html}.
-report.scm.starteam.intro = This project uses {Starteam, http://www.borland.com/us/products/starteam/} to manage its source code.
-report.scm.svn.intro = This project uses {Subversion, http://subversion.apache.org/} to manage its source code. Instructions on Subversion use can be found at {http://svnbook.red-bean.com/, http://svnbook.red-bean.com/}.
-report.scm.title = Source Code Management
-report.scm.webaccess.nourl = There is no browsable version of the source code repository listed for this project. Please check back again later.
-report.scm.webaccess.title = Web Browser Access
-report.scm.webaccess.url = The following is a link to a browsable version of the source repository:
-report.summary.build.artifactid = ArtifactId
-report.summary.build.groupid = GroupId
-report.summary.build.javaVersion = Java Version
-report.summary.build.title = Build Information
-report.summary.build.type = Type
-report.summary.build.version = Version
-report.summary.description = This document lists other related information of this project
-report.summary.field = Field
-report.summary.general.description = Description
-report.summary.general.homepage = Homepage
-report.summary.general.name = Name
-report.summary.general.title = Project Information
-report.summary.name = Summary
-report.summary.organization.name = Name
-report.summary.organization.title = Project Organization
-report.summary.organization.url = URL
-report.summary.noorganization = This project does not belong to an organization.
-report.summary.title = Project Summary
-report.summary.value = Value
-report.summary.download = Download
-report.team.contributors.actualtime = Actual Time (GMT)
-report.team.contributors.email = Email
-report.team.contributors.intro = The following additional people have contributed to this project through the way of suggestions, patches or documentation.
-report.team.contributors.image = Image
-report.team.contributors.name = Name
-report.team.contributors.organization = Organization
-report.team.contributors.organizationurl = Organization URL
-report.team.contributors.properties = Properties
-report.team.contributors.roles = Roles
-report.team.contributors.timezone = Time Zone
-report.team.contributors.title = Contributors
-report.team.contributors.url = URL
-report.team.description = This document provides information on the members of this project. These are the individuals who have contributed to the project in one form or another.
-report.team.developers.actualtime = Actual Time (GMT)
-report.team.developers.email = Email
-report.team.developers.image = Image
-report.team.developers.id = Id
-report.team.developers.intro = The following is a list of developers with commit privileges that have directly contributed to the project in one way or another.
-report.team.developers.name = Name
-report.team.developers.organization = Organization
-report.team.developers.organizationurl = Organization URL
-report.team.developers.properties = Properties
-report.team.developers.roles = Roles
-report.team.developers.timezone = Time Zone
-report.team.developers.title = Members
-report.team.developers.url = URL
-report.team.intro.description1 = A successful project requires many people to play many roles. Some members write code or documentation, while others are valuable as testers, submitting patches and suggestions.
-report.team.intro.description2 = The project team is comprised of Members and Contributors. Members have direct access to the source of a project and actively evolve the code-base. Contributors improve the project through submission of patches and suggestions to the Members. The number of Contributors to the project is unbounded. Get involved today. All contributions to the project are greatly appreciated.
-report.team.intro.title = Project Team
-report.team.name = Team
-report.team.nocontributor = There are no contributors listed for this project. Please check back again later.
-report.team.nodeveloper = There are no developers working on this project.
-report.team.title = Project Team
-report.dependency-management.name = Dependency Management
-report.dependency-management.description = This document lists the dependencies that are defined through dependencyManagement.
-report.dependency-management.title = Project Dependency Management
-report.dependency-management.nolist = There are no dependencies in the DependencyManagement of this project.
-report.dependency-management.column.groupId = GroupId
-report.dependency-management.column.artifactId = ArtifactId
-report.dependency-management.column.version = Version
-report.dependency-management.column.classifier = Classifier
-report.dependency-management.column.type = Type
-report.dependency-management.column.license = License
-report.dependency-management.intro.compile = The following is a list of compile dependencies in the DependencyManagement of this project. These dependencies are used widely across many streams modules:
-report.dependency-management.intro.provided = The following is a list of provided dependencies in the DependencyManagement of this project. These dependencies can be included in the submodules to compile the submodule, but should be provided by default when using the library:
-report.dependency-management.intro.runtime = The following is a list of runtime dependencies in the DependencyManagement of this project. These dependencies can be included in the submodules to run the submodule:
-report.dependency-management.intro.system = The following is a list of system dependencies in the DependencyManagement of this project. These dependencies can be included in the submodules to compile the submodule:
-report.dependency-management.intro.test = The following is a list of test dependencies in the DependencyManagement of this project. These dependencies can be included in the submodules to compile and run unit tests for the submodule:
-report.plugin-management.nolist = There are no plugins defined in the PluginManagement part of this project.
-report.plugin-management.name = Plugin Management
-report.plugin-management.description = This document lists the plugins that are defined through pluginManagement.
-report.plugin-management.title = Project Plugin Management
-report.plugins.name = Plugins
-report.plugins.description = This document lists the build plugins and the report plugins used by this project.
-report.plugins.title = Project Plugins
-report.plugins.build.title = Project Build Plugins
-report.plugins.report.title = Project Report Plugins
-report.plugins.nolist = There are no plugins defined in the Build part of this project.
-report.plugins.report.nolist = There are no plugins reports defined in the Reporting part of this project.
-report.modules.nolist = There are no modules declared in this project.
-report.modules.name = Project Modules
-report.modules.description = This document lists the modules (sub-projects) of this project.
-report.modules.title = Project Modules
-report.modules.intro = This project has declared the following modules:
-report.modules.header.name = Name
-report.modules.header.description = Description
-report.distribution-management.name = Distribution Management
-report.distribution-management.description = This document provides informations on the distribution management of this project.
-report.distribution-management.title = Project Distribution Management
-report.distribution-management.nodistributionmanagement = No distribution management is defined for this project.
-report.distribution-management.overview.title = Overview
-report.distribution-management.overview.intro = The following is the distribution management information used by this project.
-report.distribution-management.downloadURL = Download URL
-report.distribution-management.repository = Repository
-report.distribution-management.snapshotRepository = Snapshot Repository
-report.distribution-management.site = Site
-report.distribution-management.relocation = Relocation
-report.distribution-management.field = Field
-report.distribution-management.value = Value
-report.distribution-management.relocation.groupid = GroupId
-report.distribution-management.relocation.artifactid = ArtifactId
-report.distribution-management.relocation.version = Version
-report.distribution-management.relocation.message = Message
diff --git a/src/site/markdown/architecture.md b/src/site/markdown/architecture.md
deleted file mode 100644
index 0921b8d..0000000
--- a/src/site/markdown/architecture.md
+++ /dev/null
@@ -1,35 +0,0 @@
-## Architecture
-
-Apache Streams consists of a loosely coupled set of modules with specific capabilities, such as:
-
- - collecting data.
- - transforming or filter data
- - storing and retrieving documents and metadata from databases
- - binding streams components to other systems
- - facilitating starting and stopping of streams.
-
-
-
-#### Modules
-
-Each module has it's own POM and dependency tree. Each stream deployment needs to import only the modules it needs for what it wants to do.
-
-#### Schemas
-
-Streams also contains libraries and patterns for specifying document schemas, converting documents to and from ActivityStreams format, and generating source and resource files for binding to data objects in those formats.
-
-#### Pipelines
-
-A Pipeline is a set of collection, processing, and storage components structured in a directed graph (cycles may be permitted) which is packaged, deployed, started, and stopped together.
-
-#### Runtimes
-
-A Runtime is a module containing bindings that help setup and run a pipeline. Runtimes may submit pipeline binaries to an existing cluster, or may launch the process(es) to execute the stream directly.
-
-### Example
-
-A standard usage of Apache Streams is to collect, normalize, and archive activity across multiple networks.
-
-
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/changelog.md b/src/site/markdown/changelog.md
deleted file mode 100644
index 327b582..0000000
--- a/src/site/markdown/changelog.md
+++ /dev/null
@@ -1,560 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
- Release Notes - Streams - Version 0.4
-
-<h2> Sub-task
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-121'>STREAMS-121</a>] - InstagramTimelineProvider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-217'>STREAMS-217</a>] - Delete unneeded Configurator classes
-</li>
-</ul>
-
-<h2> Bug
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-73'>STREAMS-73</a>] - Interfaces
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-132'>STREAMS-132</a>] - RegexUtils does not ensure that content is non-null
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-151'>STREAMS-151</a>] - Refactor Facebook Provider to have continuous AND finite mode
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-159'>STREAMS-159</a>] - Add facebook page feed provider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-177'>STREAMS-177</a>] - new NPE in LinkResolver causing surefire to fail
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-258'>STREAMS-258</a>] - Prevent signing of .asc, .m5, and .sha1 artifacts
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-391'>STREAMS-391</a>] - streams-provider-dropwizard exceptions in test logs
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-405'>STREAMS-405</a>] - Link to src/site/markdown/index.md in README.md is broken
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-407'>STREAMS-407</a>] - add DateTimeSerializers for formats safely, don't crash if invalid
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-414'>STREAMS-414</a>] - Incorrect Documentation
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-436'>STREAMS-436</a>] - Put a timeout on all Provider Integration Tests
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-437'>STREAMS-437</a>] - DatumFromMetadataProcessorIT failing during 0.4 release
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-446'>STREAMS-446</a>] - RAT check fails in prep for 0.4-incubating release
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-447'>STREAMS-447</a>] - Scala-plugin failures in prep for 0.4-incubating release
-</li>
-</ul>
-
-<h2> Improvement
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-127'>STREAMS-127</a>] - JsonSchema Replication in Datasift provider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-160'>STREAMS-160</a>] - Embed original source provider pojos inside datasift pojos if possible
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-186'>STREAMS-186</a>] - Platform-level 'detectConfiguration'
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-399'>STREAMS-399</a>] - Add any missing fields to tweet.json
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-400'>STREAMS-400</a>] - streams-persist-elasticsearch : bump to version 2.x
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-403'>STREAMS-403</a>] - Ensure all providers function stand-alone
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-404'>STREAMS-404</a>] - Integration Test for FsElasticsearchIndex
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-411'>STREAMS-411</a>] - ability (and instructions on how) to run providers directly from console
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-413'>STREAMS-413</a>] - Update dependency and plugin versions - Q4 2016
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-425'>STREAMS-425</a>] - better tracking completion in multi-threaded providers
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-426'>STREAMS-426</a>] - streams-persist-mongo : test with docker
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-427'>STREAMS-427</a>] - Support any jackson-compatible class as valid input to base converters and provider converters
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-428'>STREAMS-428</a>] - Update example markdown on website
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-429'>STREAMS-429</a>] - fix failing tests in streams-plugins
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-430'>STREAMS-430</a>] - update jenkins to run advanced integration testing
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-431'>STREAMS-431</a>] - Remove streams.util.RegexUtils
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-432'>STREAMS-432</a>] - Update to Java 8
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-433'>STREAMS-433</a>] - Upgrade maven and jenkins to build with jdk8
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-434'>STREAMS-434</a>] - Delete CustomDateTimeFormat which is not used anywhere
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-435'>STREAMS-435</a>] - remove incubator-streams-master-pom.xml
-</li>
-</ul>
-
-<h2> New Feature
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-213'>STREAMS-213</a>] - Publish jsonschemas to a web-accessible URL when jenkins builds snapshot and releases
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-389'>STREAMS-389</a>] - Support generation of scala source from jsonschemas
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-398'>STREAMS-398</a>] - Support generation of hive table definitions from jsonschema
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-418'>STREAMS-418</a>] - Flink twitter example(s)
-</li>
-</ul>
-
-<h2> Story
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-51'>STREAMS-51</a>] - Complete, test, and document tika processor
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-297'>STREAMS-297</a>] - Create RssLinkProvider
-</li>
-</ul>
-
-<h2> Task
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-203'>STREAMS-203</a>] - Update GooglePlus TypeConverter to handle Post Activities
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-316'>STREAMS-316</a>] - add “apache’ to the artifact name
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-408'>STREAMS-408</a>] - Check package names and run instructions of modules in streams-examples/local
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-409'>STREAMS-409</a>] - the copyright year in the NOTICE files need to be updated for 2016.
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-410'>STREAMS-410</a>] - Delete any modules which have been removed from reactor from master branch
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-416'>STREAMS-416</a>] - Delete defunct or not-implemented provider modules
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-417'>STREAMS-417</a>] - Collect example AS 2.0 object and activity documents to use in test cases
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-419'>STREAMS-419</a>] - reboot: cleanup git branches and tags
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-421'>STREAMS-421</a>] - Delete defunct or not-implemented runtime modules
-</li>
-</ul>
-
-<h2> Test
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-415'>STREAMS-415</a>] - Proof of concept integration test that pulls actual data from generator
-</li>
-</ul>
-
- Release Notes - Streams - Version 0.3
-
-<h2> Sub-task
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-122'>STREAMS-122</a>] - Build appropriate serializer classes
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-198'>STREAMS-198</a>] - Deprecate DatumCountable
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-231'>STREAMS-231</a>] - Basic Neo4j graph persistence
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-310'>STREAMS-310</a>] - incubator-streams-examples rat plugin
-</li>
-</ul>
-
-<h2> Bug
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-158'>STREAMS-158</a>] - Sysomos Processor exceptions result in failed processor thread
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-220'>STREAMS-220</a>] - FacebookPostSerializer is broken
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-223'>STREAMS-223</a>] - streams-monitoring exception when streamConfig not set
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-227'>STREAMS-227</a>] - Array out of bounds Exception running FacebookTypeConverter
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-229'>STREAMS-229</a>] - GooglePlus TypeConverter needs to be able to accommodate String datums
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-230'>STREAMS-230</a>] - Broadcast Monitor doesn't start in all cases
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-236'>STREAMS-236</a>] - AbstractRegexExtensionExtractor should not allow duplicate entities
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-260'>STREAMS-260</a>] - FacebookPageFeedDataCollector should handle backoff strategy correctly
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-264'>STREAMS-264</a>] - LinkExpansion Tests are depending on (non-existent) external resources
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-311'>STREAMS-311</a>] - TwitterUserInformationProvider stalls with > 20 items provided
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-337'>STREAMS-337</a>] - WebHdfsPersistReader/Writer should allow user-specified file encoding
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-338'>STREAMS-338</a>] - Runtime exceptions caught by LocalStreamBuilder aren't logged
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-346'>STREAMS-346</a>] - persist-hdfs: unnecessary port config causes failure when reading from local filesystem
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-347'>STREAMS-347</a>] - persist-hdfs: incorrectly escapes output when write called on a json String
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-354'>STREAMS-354</a>] - pojo.json.Collection.totalItems should be an integer
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-355'>STREAMS-355</a>] - WebHdfsPersistReader should be serializable
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-356'>STREAMS-356</a>] - Metadata missing in StreamsDatum constructors and toString
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-357'>STREAMS-357</a>] - Streams can shutdown before starting if providers haven't started yet when runtime first checks run status
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-358'>STREAMS-358</a>] - Refreshing indexes in EsPW.cleanUp can result in streams not terminating
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-374'>STREAMS-374</a>] - Fix ActivityConverterUtil detectClasses(document) exception handling
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-377'>STREAMS-377</a>] - can't create different LineReadWriteUtils
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-380'>STREAMS-380</a>] - Media_Link should be serializable
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-382'>STREAMS-382</a>] - Media_Link width and height should be integers, not floats
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-386'>STREAMS-386</a>] - Clean up streams-pojo jsonschemas to ensure all beans are serialiazable
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-395'>STREAMS-395</a>] - incubator-streams-examples COMPILATION ERROR
-</li>
-</ul>
-
-<h2> Improvement
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-126'>STREAMS-126</a>] - StreamsLocalBuilder should look for provider timeout in typesafe config
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-191'>STREAMS-191</a>] - Streams implement use of throughput queues
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-226'>STREAMS-226</a>] - Consolidate all stream-wide configuration
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-279'>STREAMS-279</a>] - Create Youtube User/Channel provider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-294'>STREAMS-294</a>] - Allow for custom setting of QueueSize, BatchSize, and ScrollTimeout
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-308'>STREAMS-308</a>] - Update readme and website to recommend latest version of JDK
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-313'>STREAMS-313</a>] - Performant Off-line Neo4j GraphPersistWriter
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-320'>STREAMS-320</a>] - Update release documentation to cover build and deploy of mvn site to svn
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-321'>STREAMS-321</a>] - Add HOCON Converter support for input object not at root node
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-322'>STREAMS-322</a>] - Support gzipped files in WebHdfsReader
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-324'>STREAMS-324</a>] - clean up example poms
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-325'>STREAMS-325</a>] - Configure monitoring via typesafe
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-326'>STREAMS-326</a>] - Bump hadoop version in streams-persist-hdfs
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-328'>STREAMS-328</a>] - StreamsJacksonMapper should omit null/empty fields when serializing
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-329'>STREAMS-329</a>] - Processor to derive required ES metadata from standard fields
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-330'>STREAMS-330</a>] - make TwitterErrorHandler respect the Twitter4j RateLimitStatus reset time
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-332'>STREAMS-332</a>] - Restore ability to test data conversions on flat files during build
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-333'>STREAMS-333</a>] - ElasticsearchPersistUpdater should set parent+routing when available
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-335'>STREAMS-335</a>] - Publish schema and configuration resources to maven site pages
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-336'>STREAMS-336</a>] - consolidate common pom sections to streams-master
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-339'>STREAMS-339</a>] - incorporate defined extension fields for activities and actors into pojos
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-342'>STREAMS-342</a>] - Expose convertResultToString and processLine as public methods
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-343'>STREAMS-343</a>] - Make SerializationUtil spark-compatible
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-348'>STREAMS-348</a>] - Add get/set methods for StreamsConfiguration to Builder interface
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-350'>STREAMS-350</a>] - provider-twitter: derive baseURL from configuration
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-353'>STREAMS-353</a>] - Support removal of 'extensions' from document path structure
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-361'>STREAMS-361</a>] - Persist Reader/Writer for Amazon Kinesis
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-362'>STREAMS-362</a>] - Make shutdown timers in local runtime fully configurable
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-363'>STREAMS-363</a>] - upgrade persist-s3 to match persist-hdfs features
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-364'>STREAMS-364</a>] - Allow resolution of typesafe/stream config from url in stream-config
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-368'>STREAMS-368</a>] - Add String getId() to StreamsOperation
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-370'>STREAMS-370</a>] - Support arbitrary labels in streams-persist-graph
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-371'>STREAMS-371</a>] - Allow use of ids endpoints in TwitterFollowingProvider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-373'>STREAMS-373</a>] - Allow specification of 'maximum_items' in twitter providers
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-375'>STREAMS-375</a>] - Override component configuration when valid beans supplied by runtime as prepare args
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-376'>STREAMS-376</a>] - When twitter providers fail authentication, log the ID that could not be accessed.
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-378'>STREAMS-378</a>] - incompatibility between streams binaries and spark 1.5
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-381'>STREAMS-381</a>] - streams-provider-twitter: User can contain a Status
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-383'>STREAMS-383</a>] - register DefaultScalaModule in StreamsJacksonMapper
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-387'>STREAMS-387</a>] - Default behavior of streams-pojo-extensions: use additionalProperties directly
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-390'>STREAMS-390</a>] - Eliminate dependency and plugin warnings from build
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-392'>STREAMS-392</a>] - Centralized logging configuration for maven build
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-393'>STREAMS-393</a>] - Switch any usage of System.out and System.err to slf4j
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-397'>STREAMS-397</a>] - streams-provider-youtube test failure in jenkins
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-401'>STREAMS-401</a>] - Refresh Streams Website
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-406'>STREAMS-406</a>] - Bring streams-master markdown files into compliance with rat plugin
-</li>
-</ul>
-
-<h2> New Feature
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-120'>STREAMS-120</a>] - Implement, Document, and Test Instagram Provider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-221'>STREAMS-221</a>] - Simple HTTP Persist Writer
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-222'>STREAMS-222</a>] - Simple HTTP Webhook Provider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-290'>STREAMS-290</a>] - Example: local/twitter-userstream-elasticsearch
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-292'>STREAMS-292</a>] - Example: local/twitter-history-elasticsearch
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-295'>STREAMS-295</a>] - Example - elasticsearch-hdfs
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-302'>STREAMS-302</a>] - Example: local/twitter-follow-graph
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-385'>STREAMS-385</a>] - ActivityObjectConverter
-</li>
-</ul>
-
-<h2> Story
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-43'>STREAMS-43</a>] - Complete, test, and document g+ provider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-44'>STREAMS-44</a>] - Complete, test, and document sysomos provider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-46'>STREAMS-46</a>] - Complete, test, and document facebook API provider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-261'>STREAMS-261</a>] - Create Facebook Bio Collector/Provider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-272'>STREAMS-272</a>] - Create a Youtube Post Provider
-</li>
-</ul>
-
-<h2> Task
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-123'>STREAMS-123</a>] - Add twitter specific link handling for datasift
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-202'>STREAMS-202</a>] - Create Google Plus Deserializer and TypeConverter
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-274'>STREAMS-274</a>] - Create a YoutubeTypeConverter and serializer
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-280'>STREAMS-280</a>] - Add ability to get a final document count from the Sysomos Provider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-298'>STREAMS-298</a>] - Ensure jcl-over-slf4j bridge is included where necessary
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-317'>STREAMS-317</a>] - release a .tar.gz artefact as well as a .zip
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-318'>STREAMS-318</a>] - remove test files whose license may not be compatible with an Apache release (google-gplus)
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-319'>STREAMS-319</a>] - update <developer> info in the pom with active contributors
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-331'>STREAMS-331</a>] - Set up build of master and pull requested branches on travis-ci.org
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-334'>STREAMS-334</a>] - version bump datasift module to remove boundary repo dependency
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-351'>STREAMS-351</a>] - add A2 license to site.xml files
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-359'>STREAMS-359</a>] - copy streams-master pom into streams-project as CI workaround
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-372'>STREAMS-372</a>] - Support deploy to snapshot repo by CI
-</li>
-</ul>
-
-<h2> Test
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-208'>STREAMS-208</a>] - Integration Testing capability and reference Integration Test
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-384'>STREAMS-384</a>] - TestLinkUnwinderProcessor.test404Link is failing
-</li>
-</ul>
-
- Release Notes - Streams - Version 0.2
-
-<h2> Sub-task
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-275'>STREAMS-275</a>] - ActivityConverterProcessor should apply reflection mode when configuration is not provided
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-277'>STREAMS-277</a>] - Upgrade streams-provider-twitter to work with reflection-based conversion
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-305'>STREAMS-305</a>] - Add missing AL
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-306'>STREAMS-306</a>] - Intermittent test failures
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-307'>STREAMS-307</a>] - Release test-jar packaging
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-309'>STREAMS-309</a>] - incubator-streams-examples site plugin
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-312'>STREAMS-312</a>] - Remove test resources without clear licensing and ignore tests that require them
-</li>
-</ul>
-
-<h2> Bug
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-155'>STREAMS-155</a>] - Build hadoop modules with Apache artifacts
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-200'>STREAMS-200</a>] - StreamsProcessorTask ignores any processing
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-225'>STREAMS-225</a>] - Streams need to remove any of their JMX beans on shutdown/cleanup
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-243'>STREAMS-243</a>] - S3 Persist Writer does not flush or shutdown on stream shutdown
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-263'>STREAMS-263</a>] - FacebookTypeConverter should be able to handle Facebook Pages
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-266'>STREAMS-266</a>] - some classes/tests are using the wrong NotImplementedException
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-278'>STREAMS-278</a>] - Rework pig runtime as part of switch from 'Serializer' to 'Converter'
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-281'>STREAMS-281</a>] - enable BroadcastMessagePersister
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-288'>STREAMS-288</a>] - StreamsJacksonModule should not scan for DateTimeFormats by default
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-296'>STREAMS-296</a>] - Local Runtime doesn't allow persist writers enough time to flush and close during shutdown
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-299'>STREAMS-299</a>] - Sysomos Provider uses dev API URL
-</li>
-</ul>
-
-<h2> Improvement
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-147'>STREAMS-147</a>] - Platform-level type conversion
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-201'>STREAMS-201</a>] - Util function to remove all MXBeans for tests
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-214'>STREAMS-214</a>] - Create, test, and document file-backed persistance module
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-271'>STREAMS-271</a>] - suggest increasing JVM heap in readme
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-273'>STREAMS-273</a>] - Support POST endpoints in streams-http
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-284'>STREAMS-284</a>] - Read/write parent IDs in streams-persist-elasticsearch
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-285'>STREAMS-285</a>] - Add all objectTypes in spec to streams-pojo
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-286'>STREAMS-286</a>] - Add all verbs in spec to streams-pojo
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-293'>STREAMS-293</a>] - allow for missing metadata fields in streams-persist-hdfs
-</li>
-</ul>
-
-<h2> New Feature
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-235'>STREAMS-235</a>] - TwitterFollowingProvider
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-259'>STREAMS-259</a>] - Filter a stream of Activities for specific verbs and objectTypes
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-283'>STREAMS-283</a>] - Logstash-friendly monitoring
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-287'>STREAMS-287</a>] - Example: local/elasticsearch-reindex
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-289'>STREAMS-289</a>] - Example: local/mongo-elasticsearch-sync
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-301'>STREAMS-301</a>] - streams-converters: HOCON Converter
-</li>
-</ul>
-
-<h2> Story
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-47'>STREAMS-47</a>] - Complete, test, and document mongo persist
-</li>
-</ul>
-
-<h2> Task
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-95'>STREAMS-95</a>] - TwitterProfileProcessor needs to send a user ID in each StreamsDatum
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-304'>STREAMS-304</a>] - Perform 0.2-incubating release
-</li>
-</ul>
-
-
- Release Notes - Streams - Version 0.1
-
-<h2> Sub-task
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-212'>STREAMS-212</a>] - Generic Type Converter Processor
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-215'>STREAMS-215</a>] - Add method to StreamsConfigurator to return Serializable config given a Typesafe Config
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-218'>STREAMS-218</a>] - Generic Activity Converter Processor
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-241'>STREAMS-241</a>] - Reflection-based StreamsJacksonMapper
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-245'>STREAMS-245</a>] - Clean-up root POM
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-246'>STREAMS-246</a>] - Clean-up module POMs
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-247'>STREAMS-247</a>] - Clean-up READMEs
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-248'>STREAMS-248</a>] - Bring website content in-line with released capabilities.
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-249'>STREAMS-249</a>] - Javadoc plugin
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-250'>STREAMS-250</a>] - Prepare release notes
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-251'>STREAMS-251</a>] - Dry-run release process, review artifacts, update documentation
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-254'>STREAMS-254</a>] - Resolve rat plugin failures
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-255'>STREAMS-255</a>] - Merge streams-master into streams-project
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-256'>STREAMS-256</a>] - streams-components-all version doesn't bump during release
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-267'>STREAMS-267</a>] - .gitignore eclipse workspace files
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-268'>STREAMS-268</a>] - remove streams-web from master
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-269'>STREAMS-269</a>] - maven-remote-resources-plugin using SNAPSHOT resource-bundles
-</li>
-</ul>
-
-<h2> Bug
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-167'>STREAMS-167</a>] - TwitterConfigurator doesn't properly create TwitterUserInfoConfiguration from hocon
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-219'>STREAMS-219</a>] - src/main/resource files are being created by build
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-252'>STREAMS-252</a>] - Monitor Executor Service in LocalStreamBuilder needs to be Flexible
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-262'>STREAMS-262</a>] - jars in runner folder of source release
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-265'>STREAMS-265</a>] - some jar artifacts from 0.1-rc1 did not contain DISCLAIMER
-</li>
-</ul>
-
-<h2> Improvement
-</h2>
-<ul>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-68'>STREAMS-68</a>] - ActivitySerializer should be ActivityConverter
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-143'>STREAMS-143</a>] - Allow modules to get instances of StreamsJacksonMapper that can process additional DateTime formats
-</li>
-<li>[<a href='https://issues.apache.org/jira/browse/STREAMS-244'>STREAMS-244</a>] - Prepare for 0.1 release
-</li>
-</ul>
-
\ No newline at end of file
diff --git a/src/site/markdown/code-conventions.md b/src/site/markdown/code-conventions.md
deleted file mode 100644
index b72bbf4..0000000
--- a/src/site/markdown/code-conventions.md
+++ /dev/null
@@ -1,74 +0,0 @@
-Title: Apache Streams Code Conventions
-
-The Apache Streams code should follow our code conventions. All code which
-is contributed to the project should adhere to these guidelines.
-
-* Use 2 spaces for indentation. No tabs!
-* Place open braces on the same line as the declaration, for example:
- <pre><code>
-public class Foo extends Bar {
- public static void main(String args[]) {
- try {
- for (int i = 0; i < args.length; i++) {
- System.out.println(Integer.parseInt(args[i]));
- }
- }
- catch(NumberFormatException e) {
- e.printStackTrace();
- }
- }
-}
- </code></pre>
-* Wrap lines longer than 140 characters. For wrapped lines use an indent of 4 characters.
-* Within a class or interface, definitions should be ordered as follows:
- * Class (static) variables
- * Instance variables
- * Constructors
- * Methods
- * Inner classes (or interfaces)
-
-* Do not use '*' package imports (for example import org.apache.streams.*)
-* For other cases, we try to follow
- [Java code conventions](http://www.oracle.com/technetwork/java/codeconventions-150003.pdf) as much as possible.
-
-# Formatter and Style files
-* A code formatter file for IntelliJ can be found below.
-```xml
- <code_scheme name="Streams">
- <option name="CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND" value="99" />
- <option name="NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND" value="99" />
- <option name="PACKAGES_TO_USE_IMPORT_ON_DEMAND">
- <value />
- </option>
- <option name="IMPORT_LAYOUT_TABLE">
- <value>
- <package name="org.apache.streams" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="java" withSubpackages="true" static="false" />
- <package name="javax" withSubpackages="true" static="false" />
- <emptyLine />
- <package name="" withSubpackages="true" static="true" />
- </value>
- </option>
- <MarkdownNavigatorCodeStyleSettings>
- <option name="RIGHT_MARGIN" value="72" />
- </MarkdownNavigatorCodeStyleSettings>
- <codeStyleSettings language="JAVA">
- <option name="RIGHT_MARGIN" value="140" />
- <option name="SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE" value="true" />
- <option name="SPACE_BEFORE_ANNOTATION_ARRAY_INITIALIZER_LBRACE" value="true" />
- <option name="IF_BRACE_FORCE" value="3" />
- <option name="DOWHILE_BRACE_FORCE" value="3" />
- <option name="WHILE_BRACE_FORCE" value="3" />
- <option name="FOR_BRACE_FORCE" value="3" />
- <indentOptions>
- <option name="INDENT_SIZE" value="2" />
- <option name="CONTINUATION_INDENT_SIZE" value="4" />
- </indentOptions>
- </codeStyleSettings>
- </code_scheme>
- ```
-
- ###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/concepts.md b/src/site/markdown/concepts.md
deleted file mode 100644
index 8f76712..0000000
--- a/src/site/markdown/concepts.md
+++ /dev/null
@@ -1,57 +0,0 @@
-### Concepts
-
-####Component
-
-Components are the classes that do stuff within a stream. Components are assembled into pipelines and executed using a runtime. There are several core types of Components, each using a specific java interface:
-
-#####Provider
-
-A Provider is a component that *provides* data to the stream from external systems.
-
-#####Processor
-
-A Processor is a component that *processes* data flowing through the stream - transformations, filters, and enrichments are common processors.
-
-#####PersistWriter
-
-A PersistWriter is a component that *writes* data exiting the stream.
-
-#####PersistReader
-
-A PersistReader is a component that *reads* data, often previously written by a PersistWriter.
-
-####Schema
-
-A Schema defines the expected shape of the documents that will passed from step to step within a stream. Defining the schema for a type of document allows source files and resource files to be generated by the build process, relieving your team of the need to maintain these files by hand.
-
-Schemas can include other schemas, whether in the same repo or available via HTTP, allowing for full or partial reuse within or across organizations.
-
-####Datum
-
-A Datum is a single piece of data within a stream. A datum typically has an identifier, a timestamp, a document (which may be any java object), and additional metadata kept apart from the document related to upstream or downstream processing..
-
-####Activity
-
-Apache Streams has a preference for ActivityStreams formatted messages. These messages may be passed using the 'Activity' class or one of it's sub-classes.
-
-####ActivityObject
-
-An activity has several sub-object fields:
-
- - actor (required)
- - object (optional)
- - target (optional)
- - generator (optional)
- - provider (optional)
-
-Streams containing details of actors, objects, etc... may be created using the 'ActivityObject' class or one of it's sub-classes.
-
-####Pipeline
-
-A Pipeline is a set of collection, processing, and storage components structured in a directed graph (cycles may be permitted) which is packaged, deployed, started, and stopped together.
-
-####Runtime
-
-A Runtime is a module containing bindings that help setup and run a pipeline. Runtimes may submit pipeline binaries to an existing cluster, or may launch the process(es) to execute the stream directly.
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/credentials/twitter.md b/src/site/markdown/credentials/twitter.md
deleted file mode 100644
index 4960ccb..0000000
--- a/src/site/markdown/credentials/twitter.md
+++ /dev/null
@@ -1,24 +0,0 @@
-## Twitter Credentials
-
-Create a local file `twitter.oauth.conf` with valid twitter credentials
-
- twitter {
- oauth {
- consumerKey = ""
- consumerSecret = ""
- accessToken = ""
- accessTokenSecret = ""
- }
- }
-
-Log into developer.twitter.com
-
-Visit https://apps.twitter.com and create an application.
-
-Select your application and click into 'Keys and Access Tokens'
-
-The consumerKey and consumerSecret are application-wide.
-
-The accessToken and accessTokenSecret are per-user.
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/dependency-info.md b/src/site/markdown/dependency-info.md
deleted file mode 100644
index 876935c..0000000
--- a/src/site/markdown/dependency-info.md
+++ /dev/null
@@ -1,107 +0,0 @@
-##Dependency Info
-
-This project uses [Maven](http://maven.apache.org/ "Maven") for dependency management.
-
-Below are some examples of how to import Streams artifacts into your project.
-
-Please note that your project should import multiple artifacts corresponding to the Components in your stream(s) and the Runtime used to execute your stream(s).
-
-You should *not* import streams-master (depicted below), because it does not do anything interesting.
-
-Browse the "Project Modules" index of streams-project to find artifacts you might want to import.
-
-[streams-project-modules](http://streams.apache.org/site/latest/streams-project/modules.html "http://streams.apache.org/site/latest/streams-project/modules.html")
-
-<div class="section">
-
-<h3><a name="Apache_Maven"></a>Apache Maven</h3><a name="Apache_Maven"></a>
-
-<div class="source">
-
-<pre class="prettyprint"><dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-master</artifactId>
- <version>0.3-incubating-SNAPSHOT</version>
- <type>pom</type>
-</dependency></pre>
-
-</div>
-
-</div>
-
-<div class="section">
-
-<h3><a name="Apache_Buildr"></a>Apache Buildr</h3><a name="Apache_Buildr"></a>
-
-<div class="source">
-
-<pre class="prettyprint">'org.apache.streams:streams-master:pom:0.3-incubating-SNAPSHOT'</pre>
-
-</div>
-
-</div>
-
-<div class="section">
-
-<h3><a name="Apache_Ivy"></a>Apache Ivy</h3><a name="Apache_Ivy"></a>
-
-<div class="source">
-
-<pre class="prettyprint"><dependency org="org.apache.streams" name="streams-master" rev="0.3-incubating-SNAPSHOT">
- <artifact name="streams-master" type="pom" />
-</dependency></pre>
-
-</div>
-
-</div>
-
-<div class="section">
-
-<h3><a name="Groovy_Grape"></a>Groovy Grape</h3><a name="Groovy_Grape"></a>
-
-<div class="source"><pre class="prettyprint">@Grapes(
-@Grab(group='org.apache.streams', module='streams-master', version='0.3-incubating-SNAPSHOT')
-)</pre>
-
-</div>
-
-<div>
-
-<div class="section">
-
-<h3><a name="Gradle"></a>Gradle</h3><a name="Gradle"></a>
-
-<div class="source"><pre class="prettyprint">compile 'org.apache.streams:streams-master:0.3-incubating-SNAPSHOT'</pre>
-
-</div>
-
-</div>
-
-<div class="section">
-<h3><a name="Leiningen"></a>Leiningen</h3><a name="Leiningen"></a>
-
-<div class="source">
-
-<pre class="prettyprint">[org.apache.streams/streams-master "0.3-incubating-SNAPSHOT"]</pre>
-
-</div>
-
-</div>
-
-<div class="section">
-
-<h3><a name="SBT"></a>SBT</h3><a name="SBT"></a>
-
-<div class="source">
-
-<pre class="prettyprint">libraryDependencies += "org.apache.streams" % "streams-master" % "0.3-incubating-SNAPSHOT"</pre>
-
-</div>
-
-</div>
-
-</div>
-
-</div>
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/downloads.md b/src/site/markdown/downloads.md
deleted file mode 100644
index 66241ed..0000000
--- a/src/site/markdown/downloads.md
+++ /dev/null
@@ -1,10 +0,0 @@
-## Downloads
-
-All downloads can be verified using Apache Streams code signing.
-
-| Artifact | Version | Source | asc | md5 | sha1 |
-|----------|---------|
-| streams-project | 0.5-incubating | <a class="externalLink" href="http://dist.apache.org/repos/dist/release/streams/releases/0.5-incubating/streams-project-0.5-incubating-source-release.zip">zip</a> | <a class="externalLink" href="http://dist.apache.org/repos/dist/release/streams/releases/0.5-incubating/streams-project-0.5-incubating-source-release.zip.asc">asc</a> | <a class="externalLink" href="http://dist.apache.org/repos/dist/release/streams/releases/0.5-incubating/streams-project-0 [...]
-| streams-examples | 0.5-incubating | <a class="externalLink" href="http://dist.apache.org/repos/dist/release/streams/releases/0.5-incubating/streams-examples-0.5-incubating-source-release.zip">zip</a> | <a class="externalLink" href="http://dist.apache.org/repos/dist/release/streams/releases/0.5-incubating/streams-examples-0.5-incubating-source-release.zip.asc">asc</a> | <a class="externalLink" href="http://dist.apache.org/repos/dist/release/streams/releases/0.5-incubating/streams-exampl [...]
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/faq.md b/src/site/markdown/faq.md
deleted file mode 100644
index fb32c0c..0000000
--- a/src/site/markdown/faq.md
+++ /dev/null
@@ -1,138 +0,0 @@
-## Frequently Asked Questions
-
-### Why should I adopt activity streams for my project?
-
-Odds are the dataset you are working with is some combination of timestamped events and observations of entities and their relationships at various points in time. Activity Streams provides a simple yet powerful standard format for these types of data, regardless of their origin, publisher, or specific details. Activity Streams is a community-driven specification designed for interoperability and flexibility. By supporting activity streams you maximize the chance that a new data-sourc [...]
-
-### What organizations exchange data in activity streams formats?
-
-A short list of organizations and products that support activity streams format is compiled [here](who.html "here").
-
-If your organization supports activity streams, please let us know on the project mailing list.
-
-### Why should I consider using Apache Streams for my project?
-
-If you are working with structured event and or entity data that fits the Activity Streams model, and working with a JVM language, Apache Streams can simplify many of the challenging aspects involved with these types of projects. For example:
-
-* Keeping track of the original source of each piece of information
-* Harmonizing a multitude of date-time formats
-* Moving between JSON, XML, YAML, and binary serializations
-* Writing processing logic that can run in both batch and real-time workflows
-* Defining constraints and validation rules for up-stream (third-party) and in-stream (your sphere of control) data
-* Supplying run-time configuration globally and per-stream-component in a sensible manner
-
-### What does Apache Streams actually do?
-
-Apache Streams is
-
-* an SDK for data-centric JVM software
-* a set of modules that connect data-providing APIs and data-persisting analytical systems
-* a community working to make web and enterprise datasets interoperable by default
-
-Apache Streams is not
-
-* one-size-fits-all
-* prescriptive or opinionated about how it should be used
-* only useful for projects fully dedicated to activity streams datasets
-
-The primary Streams git repository apache-streams (org.apache.streams:apache-streams) contains
-
-* core interfaces and utilities
-* plugins for transforming schemas into source code and other artifacts
-* a library of modules for acquiring, transforming, and enriching data streams.
-
-Similar modules can also be hosted externally - so long as they publish maven artifacts compatible with your version of streams, you can import and use them in your streams easily.
-
-The streams community also supports a separate repository apache-streams-examples (org.apache.streams:apache-streams-examples) which contains a library of simple streams that are 'ready-to-run'. Look here to see examples of what Streams user code look like.
-
-### Why bother with any data framework at all?
-
-Why use Postgres, Elasticsearch, Cassandra, Hadoop, Linux, or Java?
-
-Frameworks make important but boring parts of systems and code just work so your team can focus on features important to your users.
-
-If you are sure you can write code that is some combination of faster, more readable, better tested, easier to learn, easier to build with, or more maintainable than any existing framework (including Streams), maybe you should.
-
-But you are probably under-estimating how difficult it will be to optimize across all of these considerations, stay current with upgrades to underlying libraries, and fix whatever bugs are discovered.
-
-Or maybe you are capable of doing it all flawlessly, but your time is just more valuable focused on your product rather than on plumbing.
-
-By joining forces with others who care about clean running water, everyone can run better, faster, stronger code assembled with more diverse expertise, tested and tuned under more use cases.
-
-### How is streams different than "*processing framework*"?
-
-You don't have to look hard to find great data processing frameworks for batch or for real-time. Pig, Hive, Storm, Spark, Samza, Flink, and Google Cloud Dataflow (soon-to-be Apache Beam) are all great. Apex and NiFi are interesting newer options. This list only includes Apache Foundation JVM projects!
-
-At the core these platforms help you connect inputs and outputs to a directed graph of computation, and run your code at scale.
-
-Streams use this computational model as well, but is more focused on intelligently and correctly modeling the data that will flow through the stream than on stream execution. In this sense Streams is an alternative to avro or protocol buffers - one which prioritizes flexibility, expressivity, interoperability, and tooling ahead of speed or compute efficiency.
-
-Streams seeks to make it easy to design and evolve streams, and to configure complex streams sensibly. Where many processing frameworks leave all business logic and configuration issues to the developer, streams modules are designed to mix-and-match. Streams modules expect to be embedded with other frameworks and are organized to make that process painless.
-
-Streams also contains a library of plug-and-play data providers to collect and normalize data from a variety of popular sources.
-
-### How do I deploy Streams?
-
-Currently you cannot deploy Streams (uppercase). Streams has no shrink-wrapped ready-to-run server process. You can however deploy streams (lowercase). The right method for packaging, deploying, and running streams depends on what runtime you are going to use.
-
-Streams includes a local runtime that uses multi-threaded execution and blocking queues within a single process. In this scenario you build an uberjar with few exclusions and ship it to a target environment however you want - maven, scp, docker, etc... You launch the stream process with an appropriate configuration and watch the magic / catastrophic fail.
-
-Alternatively, components written to streams interfaces can be bound within other platforms such as pig or spark. In this scenario, you build an uberjar that excludes the platform parts of the classpath and launch your stream using the launch style of that platform.
-
-### Can't I just dump source data directly into files or databases?
-
-Absolutely - and that will work great right up until the point where the requirements, the tools, or the way you want to index your data need to change.
-
-### What if I need data from "*specific API*"?
-
-No problem - anyone can write a Streams provider. The project contains providers that use a variety of strategies to generate near-real-time data streams, including:
-
-* sockets
-* webhooks
-* polling
-* scraping
-
-Providers can run continuously and pass-through new data, or they can work sequentially through a backlog of items. If you need to collect so many items that you can't fit all of their ids in the memory available to your stream, it's pretty simple to sub-divide your backlog into small batches and launch a series of providers for collection using frameworks such as Flink or Spark Streaming.
-
-### What if I want to keep data in "*unsupported database*"?
-
-No problem - anyone can write a Streams persist reader or persist writer. The project contains persist writers that:
-
-* write documents efficiently with batch-style binary indexing
-* write documents one-by-one to services with REST api endpoints
-* write data to local or distributed buffers.
-
-If you just want to use streams providers to collect and feed incoming data into a queueing system to work with outside of streams that's just fine.
-
-### Can't I just use "*third-party SDK*" to do the same thing?
-
-Describe any specific data collection, processing, or storage function and there are probably several if not tens of basic implementations on GitHub. There may even be language-specific libraries published by a vendor with a commercial interest in a related technology.
-
-However, in general there are a set of tradeoffs involved when relying on these packages.
-
-* They often have transitive dependencies.
-* They may not use performant HTTP and JSON libraries.
-* The object representations and lifecycle mechanisms they provide may not be consistent with the rest of your code.
-* They may source configuration properties in a problematic or cumbersome fashion.
-* Their licenses may be restrictive or undocumented.
-
-Streams goes to great lengths to regularize many of these issues so that they are uniform across project modules, and easy to reuse within new and external modules.
-
-Where quality java libraries exist, their most useful parts may be included within a streams module, with unnecessary or difficult parts of their dependency tree excluded.
-
-### Where do I start?
-
-Work your way through the 'Tutorial' menu to get up and running with streams.
-
-Then browse the 'Other Resources' menu to learn more about how streams works and why.
-
-### How can I help?
-
-* Join our mailing list.
-* Ask questions and suggest features.
-* Contribute to the documentation in one of the streams repositories.
-* Write a new provider using an existing provider as a template.
-* Add new features (and / or tests) to an existing module you intend to use.
-* Build and contributing a new example.
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/index.md b/src/site/markdown/index.md
deleted file mode 100644
index 236617f..0000000
--- a/src/site/markdown/index.md
+++ /dev/null
@@ -1,13 +0,0 @@
-## Overview
-Apache Streams unifies a diverse world of digital profiles and online activities into common formats and vocabularies, and makes these datasets accessible across a variety of databases, devices, and platforms for streaming, browsing, search, sharing, and analytics use-cases.
-
-### What is Streams?
-Apache Streams contains JRE-based modules that developers can use to easily integrate with online data sources and build polyglot indexes of activities, entities, and relationships - all based on public standards such as [Activity Streams](http://activitystrea.ms/), or other published organizational standards.
-
-### Why use Streams?
-Streams contains libraries and patterns for specifying, publishing, and inter-linking schemas, and assists with conversion of activities (posts, shares, likes, follows, etc.) and objects (profiles, pages, photos, videos, etc.) between the representation, format, and encoding preferred by supported data providers (Twitter, Instagram, etc.), and storage services (Cassandra, Elasticsearch, HBase, HDFS, Neo4J, etc.)
-
-### Why is Streams important?
-The project aims to provide simple two-way data interchange with all popular REST APIs in activity streams formats using a universal protocol. No other active open-source project has this ambitious goal, as well as production-worthy implementations for >10 services. Streams compatibility with multiple storage back-ends and ability to be embedded within any java-based real-time or batch data processing platform ensures that its interoperability features come with little technical baggage.
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/install/docker.md b/src/site/markdown/install/docker.md
deleted file mode 100644
index 996bd4e..0000000
--- a/src/site/markdown/install/docker.md
+++ /dev/null
@@ -1,24 +0,0 @@
-## Docker
-
-Run from your command line:
-
- docker version
-
-| Possible result | Explanation |
-|-----------------|-------------|
-| bash: docker: No such file or directory | You need to install docker |
-| Client: Version: < 1.0.0 | You need a newer version of docker |
-| Server: Version: < 1.0.0 | You need a newer version of docker |
-| Client: Version: > 1.0.0\nServer: Version: > 1.0.0 | You are all good |
-
-If you need to install docker, start here:
-
-[https://docs.docker.com/engine/installation/](https://docs.docker.com/engine/installation/)
-
-Run from your command line:
-
- docker ps
-
-If you see a (possibly empty) list of running containers, you are good.
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/install/git.md b/src/site/markdown/install/git.md
deleted file mode 100644
index c1aad85..0000000
--- a/src/site/markdown/install/git.md
+++ /dev/null
@@ -1,15 +0,0 @@
-#### Git
-
- git -version
-
-| Possible result | Explanation |
-|-----------------|-------------|
-| bash: git: No such file or directory | You need to install git |
-| git version < 2.7 | You should upgrade git for security reasons |
-| git version > 2.7 | You are all good |
-
-If you need to install docker, start here:
-
-[https://git-scm.com/book/en/v2/Getting-Started-Installing-Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/install/java.md b/src/site/markdown/install/java.md
deleted file mode 100644
index 8a21196..0000000
--- a/src/site/markdown/install/java.md
+++ /dev/null
@@ -1,16 +0,0 @@
-#### Java SDK
-
-Run from your command line:
-
- java -version
-
-| Possible result | Explanation |
-|-----------------|-------------|
-| Java Version >= 1.8.0u25) | You're all good |
-| Java Version < 1.8.0u25 | You need a newer JDK |
-
-If you need to install or upgrade Java, start here:
-
-[http://www.oracle.com/technetwork/java/javase/downloads/index.html](http://www.oracle.com/technetwork/java/javase/downloads/index.html)
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/install/maven.md b/src/site/markdown/install/maven.md
deleted file mode 100644
index 228f7f5..0000000
--- a/src/site/markdown/install/maven.md
+++ /dev/null
@@ -1,19 +0,0 @@
-#### Maven
-
-Run from your command line:
-
- mvn -version
-
-| Possible result | Explanation |
-|-----------------|-------------|
-| -bash: mvn: command not found | You need to install maven |
-| Error: JAVA_HOME is not defined correctly. | You need to install JDK |
-| Apache Maven >= 3.3.9+\nJava Version >= 1.8.0u25) | You're all good |
-| Apache Maven < 3.3.9 | You need a newer version of maven |
-| Java Version < 1.8.0u25 | You need a newer JDK |
-
-If you need to install maven, start here:
-
-[http://maven.apache.org/install.html](http://maven.apache.org/install.html)
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/install/sbt.md b/src/site/markdown/install/sbt.md
deleted file mode 100644
index b8bc684..0000000
--- a/src/site/markdown/install/sbt.md
+++ /dev/null
@@ -1,17 +0,0 @@
-## SBT
-
-Run from your command line:
-
- sbt
-
-| Possible result | Explanation |
-|-----------------|-------------|
-| bash: sbt: command not found | You need to install sbt |
-
-You should really install sbt-extras, like this:
-
- curl -s https://raw.githubusercontent.com/paulp/sbt-extras/master/sbt > /usr/bin/sbtx && chmod 0755 /usr/bin/sbtx
-
-Now you can easily run the streams examples using SBT, like a boss.
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/integration.md b/src/site/markdown/integration.md
deleted file mode 100644
index 05e0dc0..0000000
--- a/src/site/markdown/integration.md
+++ /dev/null
@@ -1,7 +0,0 @@
-## Continuous Integration
-
-This project uses [Jenkins](http://jenkins-ci.org/ "Jenkins") for continuous integration.
-
-
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/release-setup.md b/src/site/markdown/release-setup.md
deleted file mode 100644
index d09a093..0000000
--- a/src/site/markdown/release-setup.md
+++ /dev/null
@@ -1,74 +0,0 @@
-###Release Setup
-
-These setup steps only need to be performed on a particular machine once.
-
-Developers using Linux workstations can skip over the references to Cygwin. If using Windows, install cygwin, including Utils/gnupg and Net/openssh packages.
-
-####Create and install a SSH key
-
-1. Open a shell window. If using Windows, open a cygwin window.
-2. Use ssh-keygen to create an SSH key.
-
- ssh-keygen -t rsa -b 4096
-
- - Follow the latest steps and guides on the [ASF website](http://www.apache.org/dev/openpgp.html#generate-key) as you should **NOT** be using SHA1 and new keys **MUST** be at least 4096 bits.
- - Program defaults should be fine. No passphrase is required for the ssh key generation. The keys will be saved in ~/.ssh/id_dsa (private) and ~/.ssh/id_dsa.pub (public).
- - See [Authenticating By Public Key (OpenSSH)](http://www.networknewz.com/networknewz-10-20030707AuthenticatingbyPublicKeyOpenSSH.html) for a good description on why and how to perform this task.
-3. SCP your SSH public key ~/.ssh/id_dsa.pub created in last step to ~/id_dsa.pub on people.apache.org.
-
- cd ~/.ssh
- scp id_dsa.pub <your userid>@people.apache.org:id_dsa.pub
-
- - You will be prompted for your password.
-4. Use ssh to login to people.apache.org
-
- cd ~
- ssh <your userid>@people.apache.org
-
- - At this point, you will still be prompted for your password.
-5. Create a ~/.ssh folder in your home directory on people.apache.org and change its file mode to 700.
-
- mkdir ~/.ssh
- chmod 700 ~/.ssh
-
-6. Move or append ~/id_dsa.pub to ~/.ssh/authorized_keys and change its file mode to 600.
-
- mv ~/id_dsa.pub ~/.ssh/authorized_keys
- chmod 600 ~/.ssh/authorized_keys
-
- - Each public key in the authorized_keys spans only one line. For example: "ssh-dss AAAAB3NzaC1kc3MAAA ..... agBmmfZ9uAbSqA== dsa-key-20071107"
- - *'#' in the first column is a comment line.*
-
-7. Exit out of this ssh session.
-
-8. Start a new ssh session. No login should be required this time due to the private ssh key on your local box matching up with the public ssh key in your home directory (~/.ssh).
-
- ssh <userid>@people.apache.org
-
- - If you are still prompted for a password, then you have not set up the ssh keys properly. Review the steps above and ensure that all of the steps were followed properly. Or, maybe the instructions are still not quite right and they still need some adjusting. In that case, please update the instructions accordingly.*
-
-####Create a GPG key
-
-1. Open a shell window. If using Windows, open a cygwin window.
-2. Generate a key-pair with gpg, using default key kind ("RSA and RSA") and keys size (4096).
-
- gpg --gen-key
-
- - The program's default values should be fine. For the "Real Name" enter your full name (ie. Stan Programmer). For the "e-mail address" enter your apache address (ie. sprogrammer@apache.org). You will also be required to enter a "passphrase" for the GPG key generation. Keep track of this as you will need this for the Release processing.
- - *The generated keys are stored in $HOME/.gnupg or %HOME%\Application Data\gnupg subdirectory.*
- - *Save the content in this subdirectory to a safe media. This contains your private key used to sign all the Streams release materials.*
-
-3. Backup your home directory to another media ||
-4. Add your public key to the [KEYS](https://github.com/apache/streams/blob/master/KEYS) file. See the commands describe at the beginning of this KEYS file to perform this task. The gpg key-pair is used to sign the published artifacts for the Streams releases.
-
- gpg --list-sigs <Real Name> && gpg --armor -- export <Real Name>
-
-5. Submit your public key to a key server. E.g. [SURFNET](http://pgp.surfnet.nl:11371/) or [MIT](http://pgp.mit.edu/)
-
-6. Following the instructions in http://people.apache.org/~henkp/trust/ and ask multiple (at least 3) current Apache committers to sign your public key.
-
-####Configure Maven
-
-1. Update your ~/.m2/settings.xml with the properties from [Publishing Maven Artifacts](http://www.apache.org/dev/publishing-maven-artifacts.html#dev-env)
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/release.md b/src/site/markdown/release.md
deleted file mode 100644
index fc90791..0000000
--- a/src/site/markdown/release.md
+++ /dev/null
@@ -1,258 +0,0 @@
-##Release Process
-
-There are two distinct sets of artifacts in two repositories that are typically released together: apache-streams and apache-streams-examples.
-
-apache-streams artifacts include streams source code and convenience jars by module.
-
-apache-streams-examples source code builds ready-to-run stand-alone examples as uber-jars.
-
-Releases must be built and published in the following order:
-
-* apache-streams
-* apache-streams-examples
-
-In the instructions below, ${project.name} should be one of these top-level repository aggregator pom project names.
-
-###NOTE:
-
-As a streams release manager, you probably want to use the release.sh and publish-rc.sh scripts in the root of apache-streams.
-
-To perform a release manually, or to better understand what's going on, read the instructions below.
-
-In either case, you'll need to first refer to [Release Setup Information](/release-setup.html).
-
-###Common Release Steps
-
-1. Environment setup for releasing artifacts (same for SNAPSHOTs and releases)
-
- 1. Use the latest Sun 1.8.x JDK
- 2. Use Maven 3.3.9 or later
- 3. Make sure the [Release Setup](release-setup.html) steps have been performed.
-
-2. Prepare the source for release:
-
- 1. Cleanup JIRA so the Fix Version in issues resolved since the last release includes this release version correctly.
- 2. Update the text files in a working copy of the project root -
- 1. Update the CHANGELOG based on the Text release reports from JIRA.
- 2. Review and update README.md if needed.
- 3. Commit any changes back to git
- 3. Stage any Roadmap or Release landing pages on the site.
-
-3. Create a release candidate branch from master.
-
- X should start at 1 and increment if early release candidates fail to complete the release cycle.
-
- git checkout master
- git branch ${project.name}-${project.version}-rcX
-
-4. Verify the source has the required license headers before trying to release:
-
- mvn -Papache-release clean apache-rat:check -e -DskipTests
-
-5. Do a dry run of the release:prepare step:
-
- mvn -Papache-release release:prepare -DautoVersionSubmodules=true -DdryRun=true
-
- - The dry run will not commit any changes back to SCM and gives you the opportunity to verify that the release process will complete as expected. You will be prompted for the following information :
-
- * Release version - take the default (should be ${project.version}-incubating)
- * SCM release tag - *DO NOT TAKE THE DEFAULT* - ${project.artifactId}-${project.version}-rcX
- * New development version - take the default (should be ${project.version}-incubating-SNAPSHOT)
- * GPG Passphrase
-
- - *If you cancel a release:prepare before it updates the pom.xml versions, then use the release:clean goal to just remove the extra files that were created.*
-
- - The Maven release plugin checks for SNAPSHOT dependencies in pom's. It will not complete the prepare goal until all SNAPSHOT dependencies are resolved.
-
-6. Verify that the release process completed as expected
-
- 1. The release plugin will create pom.xml.tag files which contain the changes that would have been committed to SVN. The only differences between pom.xml.tag and it's corresponding pom.xml file should be the version number.
- 2. Check release.properties and make sure that the scm properties have the right version. Sometimes the scm location can be the previous version not the next version.
- 3. Verify signatures ([Verifying release signatures](#verify_signatures))
-
-7. Cleanup the release prepare files again:
-
- mvn -Papache-release release:clean
-
-8. Prepare the release
-
- 1. Run the "release:prepare" step for real this time. You'll be prompted for the same version information.
-
- mvn -Papache-release -U clean release:prepare -DautoVersionSubmodules=true
-
- 2. Backup (zip or tar) your local release candidate directory in case you need to rollback the release after the next step is performed.
-
-9. Perform the release
-
- * This step will create a maven staging repository and site for use in testing and voting.
-
- mvn -Papache-release -Darguments='-Dmaven.test.skip.exec=true' release:perform -Dgoals=deploy -DlocalRepoDirectory=. -DlocalCheckout=true
-
- * If your local OS userid doesn't match your Apache userid, then you'll have to also override the value provided by the OS to Maven for the site-deploy step to work. This is known to work for Linux, but not for Mac and unknown for Windows.*
-
- -Duser.name=[your_apache_uid]
-
-10. Verify the Nexus release artifacts
-
- 1. Verify the staged artifacts in the nexus repo
- * https://repository.apache.org/index.html
- * Staging repositories (under Build Promotion) --> Name column --> org.apache.streams
- * Navigate through the artifact tree and make sure that all javadoc, sources, tests, jars, ... have .asc (GPG signature) and .md5 files. See http://people.apache.org/~henkp/repo/faq.html and http://www.apache.org/dev/release-signing.html#openpgp-ascii-detach-sig
-
- 2. Close the nexus staging repo
- * https://repository.apache.org/index.html
- * Staging repositories (under Build Promotion) --> Name column --> org.apache.streams
- * Click checkbox for the open staging repo (org.apache.streams-XXX) and press Close in the menu bar.
-
-11. Put the release candidate up for a vote
-
- 1. Create a VOTE email thread on dev@ to record votes as replies
- 2. Create a DISCUSS email thread on dev@ for any vote questions
- 3. Perform a review of the release and cast your vote. See the following for more details on Apache releases
- [http://www.apache.org/dev/release.html](http://www.apache.org/dev/release.html)
- 4. A -1 vote does not necessarily mean that the vote must be redone, however it is usually a good idea to rollback the release if a -1 vote is received. See - Recovering from a vetoed release
- 5. After the vote has been open for at least 72 hours, has at least three +1 PMC votes and no -1 votes, then post the results to the vote thread by -
- * reply to the initial email and prepend to the original subject "[RESULT]"
- * Include a list of everyone who voted +1, 0 or -1.
- 6. Promote the staged nexus artifacts
- * https://repository.apache.org/index.html
- * Staging repositories (under Build Promotion) --> Name column --> org.apache.streams
- * Click checkbox of the closed staging repo (org.apache.streams-XXX) and select Release.
-
-12. Complete the release
-
- 1. Copy the source artifacts over to the distribution area
-
- svn co https://dist.apache.org/repos/dist/release/streams/releases ./streams-releases (KEEP this directory until after the release process has been completed)
- cd ./streams-releases
- mkdir ${project.version}
- cd ./${project.version}
- wget https://repository.apache.org/content/repositories/releases/org/apache/streams/${project.name}/${project.version}/${project.name}-${project.version}-source-release.zip
- wget https://repository.apache.org/content/repositories/releases/org/apache/streams/${project.name}/${project.version}/${project.name}-${project.version}-source-release.zip.asc
- wget https://repository.apache.org/content/repositories/releases/org/apache/streams/${project.name}/${project.version}/${project.name}-${project.version}-source-release.zip.md5
- wget https://repository.apache.org/content/repositories/releases/org/apache/streams/${project.name}/${project.version}/${project.name}-${project.version}-source-release.zip.sha1
- svn add ${project.name}-*
- svn commit -m "Committing Source Release for ${project.name}-${project.version}
-
- 2. Create an official release tag from the successful release candidate tag.
-
- git checkout ${project.name}-${project.version}-rcX
- git tag -a ${project.name}-${project.version} -m 'release tag ${project.name}-${project.version}'
- git push origin :refs/tags/streams-project-${project.version}
-
- 3. Update the staged website
- * Update the downloads page (downloads.md) to add new version using the mirrored URLs
- * Modify the URL for the prior release to the archived URL for the release
-
- 4. Publish the website (see [website](website.html "Website Management"))
- * WAIT 24hrs after committing releases for mirrors to replicate
- * Publish updates to the download page
-
- 5. Delete the prior versions
- * Navigate to the release directories checked out in the prior steps
- * Delete the prior release artifacts using the svn delete command
- * Commit the deletion
-
-14. Update the JIRA versions page to close all issues, mark the version as "released", and set the date to the date that the release was approved. You may also need to make a new release entry for the next release.
-
-15. Announcing the release
- * Make a news announcement on the streams homepage.
- * Make an announcement about the release on the dev@streams.apache.org, and announce@incubator.apache.org list as per the Apache Announcement Mailing Lists page)
-
-####Recovering from a vetoed release
-
-1. Reply to the initial vote email and prepend to the original subject -
-
- [CANCELED]
-
-2. Clean the release prepare files and hard reset the release candidate branch.
-
- mvn -P apache-release release:clean
-
-3. Delete the git tag created by the release:perform step -
-
- git tag -d streams-project-${project.version}-rcX
- git push origin :refs/tags/streams-project-${project.version}-rcX
-
-4. Delete the build artifacts on people & www
-
- rm -rfv /www/people.apache.org/builds/streams/${project.version}
- rm -rfv /www/www.apache.org/dist/streams/${project.version}
-
-5. Drop the nexus staging repo
-
- 1. https://repository.apache.org/index.html
- 2. Enterprise --> Staging
- 3. Staging tab --> Name column --> org.apache.streams
- 4. Right click on the closed staging repo (org.apache.streams-XXX) and select Drop.
-
-5. Remove the staged site
-
-6. Make the required updates that caused the vote to be canceled during the next release cycle
-
-<a name="verify_signatures" ></a>
-####Verifying release signatures
-On unix platforms the following command can be executed -
-
- for file in `find . -type f -iname '*.asc'`
- do
- gpg --verify ${file}
- done
-
-You'll need to look at the output to ensure it contains only good signatures -
-
-gpg: Good signature from ...
-gpg: Signature made ...
-
-
-<a name="combined" ></a>
-####Combined Release
-In order to perform a combined release of the streams-master and streams-project trunks, do the following:
-
- * Perform Steps 1-9 of the [release](#release-steps) for apache-streams and apache-streams-examples
- * Do NOT perform step 10 until steps 1-9 have been completed for BOTH projects
- * Build the streams-master FIRST
- * When prompted to change dependencies on SNAPSHOTs, do so to the corresponding releases that you just built
-
- * Execute the remaining steps using the following e-mail template
-
- to: dev@streams.apache.org
- subject: [VOTE] Apache Streams ${release.version} Release Candidate
-
- I've created a combined ${release.version} release candidate, with the
- following artifacts up for a vote:
-
- apache-streams source tag (r${release.version}):
- https://gitbox.apache.org/repos/asf?p=streams.git;a=commit;h=...
-
- apache-streams-examples source tag (r${release.version}):
- https://gitbox.apache.org/repos/asf?p=streams-examples.git;a=commit;h=...
-
- Maven staging repo:
- https://repository.apache.org/content/repositories/${release.project.repository}
- https://repository.apache.org/content/repositories/${release.examples.repository}
-
- Source releases:
- https://repository.apache.org/content/repositories/${release.project.repository}/org/apache/rave/rave-project/${release.version}/streams-project-${release.version}-incubating-source-release.zip
- https://repository.apache.org/content/repositories/${release.project.repository}/org/apache/rave/rave-project/${release.version}/streams-project-${release.version}-incubating-source-release.zip
-
- Checksums of streams-project-${release.version}-source-release.zip:
- MD5: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
- SHA1: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-
- Checksums of streams-examples-${release.version}-source-release.zip:
- MD5: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
- SHA1: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-
- Release artifacts are signed with the following key:
- https://people.apache.org/keys/committer/...
-
- Please take the time to verify the artifacts before casting your vote.
-
- Vote will be open for 72 hours.
-
- [ ] +1 approve
- [ ] +0 no opinion
- [ ] -1 disapprove (and reason why)
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/services/cassandra.md b/src/site/markdown/services/cassandra.md
deleted file mode 100644
index 23aabf4..0000000
--- a/src/site/markdown/services/cassandra.md
+++ /dev/null
@@ -1,34 +0,0 @@
-## Cassandra
-
-Start cassandra via docker with the docker maven plugin:
-
- docker -PdockerITs docker:start
-
-Confirm that cassandra is running:
-
- docker ps
-
-Confirm that host and post(s) are in property file:
-
- cat cassandra.properties
-
-Create a local file `cassandra.conf` with cluster details:
-
- cassandra {
- hosts += ${cassandra.tcp.host}
- port = ${cassandra.tcp.port}
- }
-
-When configuring a stream, include these files:
-
- include "cassandra.properties"
- include "cassandra.conf"
-
-Supply application-specific configuration as well:
-
- cassandra {
- keyspace: ""
- table: ""
- }
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/services/elasticsearch.md b/src/site/markdown/services/elasticsearch.md
deleted file mode 100644
index 58d096b..0000000
--- a/src/site/markdown/services/elasticsearch.md
+++ /dev/null
@@ -1,35 +0,0 @@
-## Elasticsearch
-
-Start elasticsearch via docker with the docker maven plugin:
-
- docker -PdockerITs docker:start
-
-Confirm that elasticsearch is running:
-
- docker ps
-
-Confirm that host and post(s) are in property file:
-
- cat elasticsearch.properties
-
-Create a local file `elasticsearch.conf` with cluster details:
-
- elasticsearch {
- hosts += ${es.tcp.host}
- port = ${es.tcp.port}
- clusterName = "elasticsearch"
- }
-
-When configuring a stream, include these files:
-
- include "elasticsearch.properties"
- include "elasticsearch.conf"
-
-Supply application-specific configuration as well:
-
- elasticsearch {
- index: ""
- type: ""
- }
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/services/mongo.md b/src/site/markdown/services/mongo.md
deleted file mode 100644
index c2b182d..0000000
--- a/src/site/markdown/services/mongo.md
+++ /dev/null
@@ -1,34 +0,0 @@
-## Mongo
-
-Start mongo via docker with the docker maven plugin:
-
- docker -PdockerITs docker:start
-
-Confirm that elasticsearch is running:
-
- docker ps
-
-Confirm that host and post(s) are in property file:
-
- cat mongo.properties
-
-Create a local file `elasticsearch.conf` with cluster details:
-
- mongo {
- host = ${mongo.tcp.host}
- port = ${mongo.tcp.port}
- }
-
-When configuring a stream, include these files:
-
- include "mongo.properties"
- include "mongo.conf"
-
-Supply application-specific configuration as well:
-
- mongo {
- db: "",
- collection: ""
- }
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/services/neo4j.md b/src/site/markdown/services/neo4j.md
deleted file mode 100644
index 1d88ed0..0000000
--- a/src/site/markdown/services/neo4j.md
+++ /dev/null
@@ -1,29 +0,0 @@
-## Neo4j
-
-Start mongo via docker with the docker maven plugin:
-
- docker -PdockerITs docker:start
-
-Confirm that elasticsearch is running:
-
- docker ps
-
-Confirm that host and post(s) are in property file:
-
- cat neo4j.properties
-
-Create a local file `neo4j.conf` with cluster details:
-
- neo4j {
- hostname = ${neo4j.tcp.host}
- port = ${neo4j.tcp.port}
- type = "neo4j"
- graph = "data"
- }
-
-When configuring a stream, include these files:
-
- include "neo4j.properties"
- include "neo4j.conf"
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/source-repository.md b/src/site/markdown/source-repository.md
deleted file mode 100644
index da7294a..0000000
--- a/src/site/markdown/source-repository.md
+++ /dev/null
@@ -1,17 +0,0 @@
-## Source Repositories
-
-This project uses [Git](http://git.org/ "Git") for source control.
-
-Anyone is permitted to open pull requests via github.com
-
-However, only project committers are permitted to merge pull requests.
-
-###apache-streams
-
-[apache-streams](http://streams.apache.org/site/latest/streams-project "apache-streams") source is officially hosted at [github.com/apache/streams.git](https://github.com/apache/streams.git "https://github.com/apache/streams.git")
-
-###apache-streams-examples
-
-[apache-streams-examples](http://streams.apache.org/site/latest/streams-examples "apache-streams-examples") source is officially hosted at [github.com/apache/streams-examples.git](https://github.com/apache/streams-examples.git "https://github.com/apache/streams-examples.git")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/website.md b/src/site/markdown/website.md
deleted file mode 100644
index 7f70687..0000000
--- a/src/site/markdown/website.md
+++ /dev/null
@@ -1,189 +0,0 @@
-## Website Management
-
-http://streams.apache.org is a set of interconnected websites generated from markdown
-by the maven site plugin. The front page, this page, and all pages aside from those
-organized under 'Examples' reside in http://github.com/apache/streams. Those pages reside in
-http://github.com/apache/streams-examples
-
-### Website Content
-
-Pages, diagrams, and other hard-linkable resources are stored throughout the source tree.
-
-#### Markdown
-
-* src/site/markdown
-
-Most HTML pages are generated from markdown. The maven-site-plugin does this during the site build process.
-
-#### Schemas
-
-* src/main/jsonschema
-* src/main/xmlschema
-
-JSON and XML Schemas through-out the project are copied to the web page of their respective modules so they can be
-linked to in other projects.
-
-This allows users to extend the canonical streams schemas incrementally and/or re-use specific fields as they see fit.
-
-#### Resources
-
-* src/main/resources
-* src/test/resources
-
-Other resources including .conf and .properties files are copied to the web page of their respective modules so
-they can be linked to across projects and in external projects.
-
-This allows users to import HOCON from modules outside their sphere of control and adapt to changes upstream.
-
-### Website Updates
-
-The project website(s) are hosted by the Apache foundation and updated via SVN.
-
-Currently pushing website changes is a manual process with several steps, performed by whomever is making the change.
-
-This typically requires checking out the current website from SVN.
-
- svn co https://svn.apache.org/repos/asf/streams/site/trunk/content
- cd content
-
-NOTE:
-
-Repositories should always be built and published in the following order:
-
-* streams-project
-* streams-examples
-
-#### Preparing to publishing a new website version
-
-The instructions below presume:
-
-* you have a shell open in the SVN content directory
-* you know the artifactId and version of the repository you want to publish.
-
-If this is a brand new snapshot or release version, you first need to create a directory corresponding to the new version.
-
- mkdir site/${project.version}
- svn add site/${project.version}
- svn commit -m "svn add site/${project.version}"
-
-The first time a specific site is being published for this version, you must create the directory where it will be published.
-
- mkdir site/${project.version}/${project.artifactId}
- svn add site/${project.version}/${project.artifactId}
- svn commit -m "svn add site/${project.version}/${project.artifactId}"
-
-If you are published over an existing snapshot, you may need to first remove the existing version and recreate an empty directory.
-
- rm -rf site/${project.version}/${project.artifactId}
- svn delete site/${project.version}/${project.artifactId}
- svn commit -m "svn delete site/${project.version}/${project.artifactId}"
- mkdir site/${project.version}/${project.artifactId}
- svn add site/${project.version}/${project.artifactId}
- svn commit -m "svn add site/${project.version}/${project.artifactId}"
-
-In general however the scm plugin is smart enough to just add new resources and commit changes.
-
-If you are publishing a release, it's appropriate to delete the site snapshots related to the prior releases.
-
-For example when 0.3-incubating is published, 0.2-incubating-SNAPSHOT should be deleted.
-
-This policy of removing old snapshots keeps external projects from linking to snapshot artifacts indefinitely.
-
-Release artifacts should be retained indefinitely.
-
-#### Generating and committing a new website version
-
-The instructions below presume:
-
-* you have a shell open in the root of a project repository
-* you know the artifactId and version of the repository you want to publish.
-
-First, ensure that you have local credentials capable of publishing the site.
-
- <server>
- <id>site.streams.{master|project|examples}</id>
- <username>{your apache ID}</username>
- <privateKey>{absolute path to your private key</privateKey>
- <passphrase>{your private key passphrase}</passphrase>
- <filePermissions>664</filePermissions>
- <directoryPermissions>775</directoryPermissions>
- <configuration></configuration>
- </server>
-
-Next, generate SVG resources for all DOT diagrams in the source tree
-
- for dot in $(find . -name *.dot); do dot -Tsvg $dot -o $dot.svg; done
-
-Then, generate the site that will be published
-
- mvn clean generate-sources package -Dmaven.test.skip.exec=true site:site site:stage
-
-Double-check the logs and determine where exactly the staged site is located on your local drive.
-
-At this point you can open target/staging/index.html (or wherever) and do a basic sanity check on the site you intend to publish.
-
-Finally, publish the site.
-
- mvn scm-publish:publish-scm
-
-You may need to provide -Dscmpublish.content=<> depending exactly where the staging site directory winds up.
-
-Note the revision number checked in at the bottom of the maven logs.
-
-#### Updating the staging site
-
-Next step is to update the staging site and check it out.
-
-Log into https://cms.apache.org with your apache credentials.
-
-Use https://cms.apache.org/streams to access the streams website.
-
-Typically you can use Get streams Working Copy, although you might need to Force if you run into conflicts in SVN.
-
-You'll probably need to 'Update this directory' if you want to inspect the changes you committed above.
-
-'View Staging Builds' should show a build around the time of the previous commit. This means the change has been staged.
-
-You should now be able to access and review the published site(s) via the staging URL:
-
-* http://streams.staging.apache.org/
-
-At this point use explicit versions to access and review the new documentation, i.e.
-
-* http://streams.staging.apache.org/site/0.3-incubating-SNAPSHOT/streams-project/index.html
-* http://streams.staging.apache.org/site/0.3-incubating-SNAPSHOT/streams-examples/index.html
-
-#### Managing version pointers
-
-When new versions of these sites are built for the first time, an additional set to alter Apache rules may be appropriate.
-
-The convention we use exposes the latest specific site version(s) using redirects maintained in the .htaccess file of project website SVN.
-
-These rules are located in the '.htaccess' file in the root of the SVN content directory and looks something like this:
-
- Options +FollowSymLinks
- RewriteEngine on
- RedirectMatch "^/$" "/site/0.3-incubating-SNAPSHOT/streams-project"
- Redirect /site/latest /site/0.3-incubating-SNAPSHOT
- Redirect /site/latest/streams-project /site/0.3-incubating-SNAPSHOT/streams-project
- Redirect /site/latest/streams-examples /site/0.3-incubating-SNAPSHOT/streams-examples
-
-Adjust the Redirect rules as appropriate for the project and version you are deploying. You can do this directly from the CMS.
-
-Commit your changes, wait a few seconds, click Follow Staging Build, and you should see a new build with a 'Build Successful' message.
-
-You should now be able to use 'latest' as an alias for the docs you are deploying, i.e.
-
-* http://streams.staging.apache.org/site/latest/streams-project/index.html
-* http://streams.staging.apache.org/site/latest/streams-examples/index.html
-
-#### Promoting a new website version
-
-All that's left at this point is to promote from staging to production.
-
-If a release is happening, this should happen just prior to the release announcement.
-
-If you are just updating content associated with a snapshot, use good judgement to determine whether the list should have a chance to review
-and/or vote on the changes in staging prior to promotion.
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/markdown/who.md b/src/site/markdown/who.md
deleted file mode 100644
index 4add70f..0000000
--- a/src/site/markdown/who.md
+++ /dev/null
@@ -1,12 +0,0 @@
-### Who uses activity streams?
-
-| Organization (Product) | Reference |
-|------------------------|-----------|
-| Atlassian (JIRA) | [Consuming an Activity Streams Feed](https://developer.atlassian.com/docs/atlassian-platform-common-components/activity-streams/consuming-an-activity-streams-feed "Consuming an Activity Streams Feed") |
-| Google (Google Plus) | [Google+ Platform for Web](https://developers.google.com/+/web/api/rest/latest/activities#resource-representations "Google+ Platform for Web") |
-| IBM | [Activity Streams](https://developer.ibm.com/open/activity-streams/ "Activity Streams") |
-| Twitter (Gnip) | [Activity Streams Intro](http://support.gnip.com/articles/activity-streams-intro.html "Activity Streams Intro") |
-
-If your organization supports activity streams, please let us know on the project mailing list.
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/site/site.xml b/src/site/site.xml
deleted file mode 100644
index bc87ee5..0000000
--- a/src/site/site.xml
+++ /dev/null
@@ -1,124 +0,0 @@
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project>
- <publishDate position="bottom"/>
- <version position="bottom"/>
- <skin>
- <groupId>org.apache.maven.skins</groupId>
- <artifactId>maven-fluido-skin</artifactId>
- <version>1.5</version>
- </skin>
- <bannerLeft>
- <name>Apache Streams</name>
- <src>http://streams.apache.org/images/streams_logo.jpg</src>
- <href>http://streams.apache.org</href>
- <height>100</height>
- <width>150</width>
- </bannerLeft>
- <custom>
- <fluidoSkin>
- <topBarEnabled>true</topBarEnabled>
- <navBarStyle>navbar-inverse</navBarStyle>
- <sideBarEnabled>false</sideBarEnabled>
- <gitHub>
- <projectId>apache/streams</projectId>
- <ribbonOrientation>right</ribbonOrientation>
- <ribbonColor>black</ribbonColor>
- </gitHub>
- <twitter>
- <user>ApacheStreams</user>
- <showUser>true</showUser>
- <showFollowers>true</showFollowers>
- </twitter>
- </fluidoSkin>
- </custom>
- <body>
- <breadcrumbs>
- <item name="Streams" href="http://streams.apache.org/site/latest/streams-project"/>
- </breadcrumbs>
- <head>
- <![CDATA[
- <script>
- (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
- (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
- m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
- })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
-
- ga('create', 'UA-77087101-1', 'auto');
- ga('send', 'pageview');
- </script>
- ]]>
- </head>
- <menu name="Overview" inherit="top">
- <item name="Overview" href="index.html" />
- <item name="Architecture" href="architecture.html" />
- <item name="Concepts" href="concepts.html" />
- <item name="Downloads" href="downloads.html" />
- <item name="Examples" href="http://streams.apache.org/site/latest/streams-examples/" />
- <item name="Frequently Asked Questions" href="faq.html" />
- <item name="Modules" href="modules.html" />
- </menu>
- <menu name="Users">
- <item name="Install Java" href="install/java.html"/>
- <item name="Install Docker" href="install/docker.html"/>
- <item name="Install SBT" href="install/sbt.html"/>
- <item name="Credentials">
- <item name="Twitter" href="credentials/twitter.html" />
- </item>
- <item name="Services">
- <item name="Cassandra" href="services/cassandra.html" />
- <item name="Elasticsearch" href="services/elasticsearch.html" />
- <item name="Mongo" href="services/mongo.html" />
- <item name="Neo4j" href="services/neo4j.html" />
- </item>
- </menu>
- <menu name="Developers">
- <item name="Install Git" href="install/git.html"/>
- <item name="Install Maven" href="install/maven.html"/>
- <item name="Changelog" href="changelog.html" />
- <item name="Release Setup" href="release-setup.html" />
- <item name="Release Process" href="release.html" />
- <item name="Website Management" href="website.html" />
- <item name="Coding Conventions" href="code-conventions.html"/>
- </menu>
- <!--<menu name="Details">-->
- <!--<item name="Mailing Lists" href="mail-lists.html" />-->
- <!--<item name="Project Team" href="team-list.html" />-->
- <!--<item name="Continuous Integration" href="integration.html"/>-->
- <!--<item name="Issue Tracking" href="issue-tracking.html" />-->
- <!--<item name="Source Repositories" href="source-repository.html" />-->
- <!--<item name="Dependency Info" href="dependency-info.html" />-->
- <!--<item name="Dependency Management" href="dependency-management.html" />-->
- <!--<item name="Distribution Management" href="distribution-management.html" />-->
- <!--<item name="Plugin Management" href="plugin-management.html" />-->
- <!---->
- <!--<item name="Project License" href="license.html" />-->
- <!--</menu>-->
- <menu name="Reports" ref="reports" inherit="top"/>
- <menu name="Other Resources">
- <item name="ActivityStreams Homepage" href="http://activitystrea.ms" />
- <item name="ActivityStreams Specifications" href="http://activitystrea.ms/specs/" />
- <item name="Streams Blog" href="http://blogs.apache.org/streams/" />
- <item name="Streams Proposal Wiki Page" href="http://wiki.apache.org/incubator/StreamsProposal" />
- <item name="Apache Foundation Info" href="http://www.apache.org/" />
- <item name="Sponsorship" href="http://www.apache.org/foundation/sponsorship.html" />
- <item name="Thanks" href="http://www.apache.org/foundation/thanks.html" />
- </menu>
- </body>
-</project>
\ No newline at end of file
diff --git a/streams-components/README.md b/streams-components/README.md
deleted file mode 100644
index 3dc671d..0000000
--- a/streams-components/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-streams-components
-==============
-
-These modules contain generic providers, processors, and writers which
-can fulfill a variety of purposes based on the configuration they receive.
-
-They have been designed to be extended in cases where a specific integration
-needs to do something a bit different.
-
-Please help improve these packages, or create new ones!
diff --git a/streams-components/pom.xml b/streams-components/pom.xml
deleted file mode 100644
index a325c48..0000000
--- a/streams-components/pom.xml
+++ /dev/null
@@ -1,72 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>apache-streams</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-components</artifactId>
-
- <packaging>pom</packaging>
-
- <name>streams-components</name>
- <description>Functional components to embed in streams, or extend to customize behavior.</description>
-
- <modules>
- <module>streams-converters</module>
- <module>streams-filters</module>
- <module>streams-http</module>
- </modules>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-verbs</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- </dependencyManagement>
-</project>
diff --git a/streams-components/streams-converters/README.md b/streams-components/streams-converters/README.md
deleted file mode 100644
index 266a861..0000000
--- a/streams-components/streams-converters/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-converters
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-components/streams-converters/pom.xml b/streams-components/streams-converters/pom.xml
deleted file mode 100644
index 1bdb22f..0000000
--- a/streams-components/streams-converters/pom.xml
+++ /dev/null
@@ -1,137 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
- <artifactId>streams-converters</artifactId>
- <version>0.5.1-SNAPSHOT</version>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-components</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path-assert</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.converter</targetPackage>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-</project>
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityConverterProcessor.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityConverterProcessor.java
deleted file mode 100644
index b116998..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityConverterProcessor.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.core.util.DatumUtils;
-import org.apache.streams.pojo.json.Activity;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * ActivityConverterProcessor is a utility processor for converting any datum document
- * to an Activity.
- *
- * <p/>
- * By default it will handle string json and objectnode representation of existing Activities,
- * translating them into the POJO representation(s) preferred by each registered/detected
- * ActivityConverter.
- *
- * <p/>
- * To use this capability without a dedicated stream processor, just use ActivityConverterUtil.
- */
-public class ActivityConverterProcessor implements StreamsProcessor {
-
- public static final String STREAMS_ID = "ActivityConverterProcessor";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ActivityConverterProcessor.class);
-
- private ActivityConverterUtil converterUtil;
-
- private ActivityConverterProcessorConfiguration configuration;
-
- public ActivityConverterProcessor() {
- }
-
- public ActivityConverterProcessor(ActivityConverterProcessorConfiguration configuration) {
- this.configuration = configuration;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = new LinkedList<>();
- Object document = entry.getDocument();
-
- try {
-
- // first determine which classes this document might actually be
- List<Activity> activityList = converterUtil.convert(document);
-
- for (Activity activity : activityList) {
- StreamsDatum datum = DatumUtils.cloneDatum(entry);
- datum.setId(activity.getId());
- datum.setDocument(activity);
- result.add(datum);
- }
-
- } catch (Exception ex) {
- LOGGER.warn("General exception in process! " + ex.getMessage());
- }
-
- return result;
-
- }
-
- @Override
- public void prepare(Object configurationObject) {
- if (configurationObject instanceof ActivityConverterProcessorConfiguration) {
- converterUtil = ActivityConverterUtil.getInstance((ActivityConverterProcessorConfiguration) configurationObject);
- } else {
- converterUtil = ActivityConverterUtil.getInstance();
- }
- }
-
- @Override
- public void cleanUp() {
-
- }
-
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityConverterUtil.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityConverterUtil.java
deleted file mode 100644
index c535710..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityConverterUtil.java
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter;
-
-import org.apache.streams.data.ActivityConverter;
-import org.apache.streams.data.DocumentClassifier;
-import org.apache.streams.data.util.ActivityUtil;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.Activity;
-
-import com.google.common.base.Preconditions;
-import org.reflections.Reflections;
-import org.reflections.scanners.SubTypesScanner;
-import org.reflections.util.ClasspathHelper;
-import org.reflections.util.ConfigurationBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-
-/**
- * ActivityConverterUtil converts document into all possible Activity
- * representations based on registered DocumentClassifiers and ActivityConverters.
- *
- * <p/>
- * Implementations and contributed modules may implement DocumentClassifiers
- * and ActivityConverters to translate additional document types into desired
- * Activity formats.
- *
- * <p/>
- * A DocumentClassifier's reponsibility is to recognize document formats and label them,
- * using a jackson-compatible POJO class.
- *
- * <p/>
- * An ActivityConverter's reponsibility is to converting a raw document associated with an
- * incoming POJO class into an activity.
- *
- */
-public class ActivityConverterUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ActivityConverterUtil.class);
-
- private static final ActivityConverterUtil INSTANCE = new ActivityConverterUtil();
-
- public static ActivityConverterUtil getInstance() {
- return INSTANCE;
- }
-
- public static ActivityConverterUtil getInstance(ActivityConverterProcessorConfiguration configuration) {
- return new ActivityConverterUtil(configuration);
- }
-
- private List<DocumentClassifier> classifiers = new LinkedList<>();
- private List<ActivityConverter> converters = new LinkedList<>();
-
- /*
- Use getInstance to get a globally shared thread-safe ActivityConverterUtil,
- rather than call this constructor. Reflection-based resolution of
- converters across all modules can be slow and should only happen
- once per JVM.
- */
- protected ActivityConverterUtil() {
- configure();
- }
-
- protected ActivityConverterUtil(ActivityConverterProcessorConfiguration configuration) {
- classifiers = configuration.getClassifiers();
- converters = configuration.getConverters();
- configure();
- }
-
- /**
- * convert document to activity.
- *
- * @param document document to convert
- * @return result
- */
- public List<Activity> convert(Object document) {
-
- List<Activity> result = new ArrayList<>();
-
- List<Class> detectedClasses = detectClasses(document);
-
- if ( detectedClasses.size() == 0 ) {
- LOGGER.warn("Unable to classify");
- return null;
- } else {
- LOGGER.debug("Classified document as " + detectedClasses);
- }
-
- // for each of these classes:
- // use TypeUtil to switch the document to that type
- Map<Class, Object> typedDocs = convertToDetectedClasses(detectedClasses, document);
-
- if ( typedDocs.size() == 0 ) {
- LOGGER.warn("Unable to convert to any detected Class");
- return null;
- } else {
- LOGGER.debug("Document has " + typedDocs.size() + " representations: " + typedDocs.toString());
- }
-
- // for each specified / discovered converter
- for ( ActivityConverter converter : converters ) {
-
- Object typedDoc = typedDocs.get(converter.requiredClass());
-
- List<Activity> activities = applyConverter(converter, typedDoc);
-
- result.addAll(activities);
- }
-
- return result;
- }
-
- protected List<Activity> applyConverter(ActivityConverter converter, Object typedDoc) {
-
- List<Activity> activities = new ArrayList<>();
- // if the document can be typed as the required class
- if ( typedDoc != null ) {
-
- // let the converter create activities if it can
- try {
- activities = convertToActivity(converter, typedDoc);
- } catch ( Exception ex ) {
- LOGGER.debug("convertToActivity caught exception " + ex.getMessage());
- }
-
- }
- return activities;
- }
-
- protected List<Activity> convertToActivity(ActivityConverter converter, Object document) {
-
- List<Activity> activities = new ArrayList<>();
- try {
- activities = converter.toActivityList(document);
- } catch (ActivityConversionException e1) {
- LOGGER.debug(converter.getClass().getCanonicalName() + " unable to convert " + converter.requiredClass().getClass().getCanonicalName() + " to Activity");
- }
-
- for (Activity activity : activities) {
-
- if (activity != null) {
-
- // only accept valid activities
- // this primitive validity check should be replaced with
- // one that applies javax.validation to JSR303 annotations
- // on the Activity json schema once a suitable implementation
- // is found.
- if (!ActivityUtil.isValid(activity)) {
- activities.remove(activity);
- LOGGER.debug(converter.getClass().getCanonicalName() + " produced invalid Activity converting " + converter.requiredClass().getClass().getCanonicalName());
- }
-
- } else {
- LOGGER.debug(converter.getClass().getCanonicalName() + " returned null converting " + converter.requiredClass().getClass().getCanonicalName() + " to Activity");
- }
-
- }
- return activities;
-
- }
-
- protected List<Class> detectClasses(Object document) {
-
- // ConcurrentHashSet is preferable, but it's only in guava 15+
- // spark 1.5.0 uses guava 14 so for the moment this is the workaround
- // Set<Class> detectedClasses = new ConcurrentHashSet();
- Set<Class> detectedClasses = Collections.newSetFromMap(new ConcurrentHashMap<Class, Boolean>());
-
- for ( DocumentClassifier classifier : classifiers ) {
- try {
- List<Class> detected = classifier.detectClasses(document);
- if (detected != null && detected.size() > 0) {
- detectedClasses.addAll(detected);
- }
- } catch ( Exception ex ) {
- LOGGER.warn("{} failed in method detectClasses - ()", classifier.getClass().getCanonicalName(), ex);
- }
- }
-
- return new ArrayList<>(detectedClasses);
- }
-
- private Map<Class, Object> convertToDetectedClasses(List<Class> datumClasses, Object document) {
-
- Map<Class, Object> convertedDocuments = new HashMap<>();
- for ( Class detectedClass : datumClasses ) {
-
- Object typedDoc;
- if (detectedClass.isInstance(document)) {
- typedDoc = document;
- } else {
- typedDoc = TypeConverterUtil.getInstance().convert(document, detectedClass);
- }
-
- if ( typedDoc != null ) {
- convertedDocuments.put(detectedClass, typedDoc);
- }
- }
-
- return convertedDocuments;
- }
-
- /**
- * configure ActivityConverterUtil.
- */
- public void configure() {
- Reflections reflections = new Reflections(new ConfigurationBuilder()
- .setUrls(ClasspathHelper.forPackage("org.apache.streams"))
- .setScanners(new SubTypesScanner()));
- if ( classifiers.size() == 0) {
- Set<Class<? extends DocumentClassifier>> classifierClasses = reflections.getSubTypesOf(DocumentClassifier.class);
- for (Class classifierClass : classifierClasses) {
- try {
- this.classifiers.add((DocumentClassifier) classifierClass.newInstance());
- } catch (Exception ex) {
- LOGGER.warn("Exception instantiating " + classifierClass);
- }
- }
- }
- Preconditions.checkArgument(classifiers.size() > 0);
- if ( converters.size() == 0) {
- Set<Class<? extends ActivityConverter>> converterClasses = reflections.getSubTypesOf(ActivityConverter.class);
- for (Class converterClass : converterClasses) {
- try {
- this.converters.add((ActivityConverter) converterClass.newInstance());
- } catch (Exception ex) {
- LOGGER.warn("Exception instantiating " + converterClass);
- }
- }
- }
- Preconditions.checkArgument(this.converters.size() > 0);
- }
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityObjectConverterProcessor.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityObjectConverterProcessor.java
deleted file mode 100644
index bb348b6..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityObjectConverterProcessor.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.core.util.DatumUtils;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * ActivityObjectConverterProcessor is a utility processor for converting any datum document
- * to an Activity.
- *
- * <p/>
- * By default it will handle string json and objectnode representation of existing Activities,
- * translating them into the POJO representation(s) preferred by each registered/detected
- * ActivityConverter.
- *
- * <p/>
- * To use this capability without a dedicated stream processor, just use ActivityConverterUtil.
- */
-public class ActivityObjectConverterProcessor implements StreamsProcessor {
-
- public static final String STREAMS_ID = "ActivityObjectConverterProcessor";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ActivityObjectConverterProcessor.class);
-
- private ActivityObjectConverterUtil converterUtil;
-
- private ActivityObjectConverterProcessorConfiguration configuration;
-
- public ActivityObjectConverterProcessor() {
- }
-
- public ActivityObjectConverterProcessor(ActivityObjectConverterProcessorConfiguration configuration) {
- this.configuration = configuration;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = new LinkedList<>();
- Object document = entry.getDocument();
-
- try {
-
- // first determine which classes this document might actually be
- ActivityObject activityObject = converterUtil.convert(document);
-
- StreamsDatum datum = DatumUtils.cloneDatum(entry);
- datum.setId(activityObject.getId());
- datum.setDocument(activityObject);
- result.add(datum);
-
- } catch ( Exception ex ) {
- LOGGER.warn("General exception in process! " + ex.getMessage());
- }
-
- return result;
-
- }
-
- @Override
- public void prepare(Object configurationObject) {
- if( configurationObject instanceof ActivityObjectConverterProcessorConfiguration) {
- converterUtil = ActivityObjectConverterUtil.getInstance((ActivityObjectConverterProcessorConfiguration) configurationObject);
- } else {
- converterUtil = ActivityObjectConverterUtil.getInstance();
- }
- }
-
- @Override
- public void cleanUp() {
-
- }
-
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityObjectConverterUtil.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityObjectConverterUtil.java
deleted file mode 100644
index 49cdfcb..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityObjectConverterUtil.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter;
-
-import org.apache.streams.data.ActivityObjectConverter;
-import org.apache.streams.data.DocumentClassifier;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.google.common.base.Preconditions;
-import org.reflections.Reflections;
-import org.reflections.scanners.SubTypesScanner;
-import org.reflections.util.ClasspathHelper;
-import org.reflections.util.ConfigurationBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * ActivityObjectConverterUtil converts document into all possible ActivityObject
- * representations based on registered DocumentClassifiers and ActivityObjectConverters.
- *
- * <p/>
- * Implementations and contributed modules may implement DocumentClassifiers
- * and ActivityObjectConverters to translate additional document types into desired
- * ActivityObject formats.
- *
- * <p/>
- * A DocumentClassifier's reponsibility is to recognize document formats and label them,
- * using a jackson-compatible POJO class.
- *
- * <p/>
- * An ActivityObjectConverter's reponsibility is to converting a raw document associated with an
- * incoming POJO class into an activity object.
- *
- */
-public class ActivityObjectConverterUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ActivityObjectConverterUtil.class);
-
- private static final ActivityObjectConverterUtil INSTANCE = new ActivityObjectConverterUtil();
-
- public static ActivityObjectConverterUtil getInstance() {
- return INSTANCE;
- }
-
- public static ActivityObjectConverterUtil getInstance(ActivityObjectConverterProcessorConfiguration configuration) {
- return new ActivityObjectConverterUtil(configuration);
- }
-
- private List<DocumentClassifier> classifiers = new LinkedList<>();
- private List<ActivityObjectConverter> converters = new LinkedList<>();
-
- /*
- Use getInstance to get a globally shared thread-safe ActivityConverterUtil,
- rather than call this constructor. Reflection-based resolution of
- converters across all modules can be slow and should only happen
- once per JVM.
- */
- protected ActivityObjectConverterUtil() {
- configure();
- }
-
- protected ActivityObjectConverterUtil(ActivityObjectConverterProcessorConfiguration configuration) {
- classifiers = configuration.getClassifiers();
- converters = configuration.getConverters();
- configure();
- }
-
- /**
- * convert document to ActivityObject.
- *
- * @param document document to convert
- * @return result
- */
- public synchronized ActivityObject convert(Object document) {
-
- List<Class> detectedClasses = detectClasses(document);
-
- if (detectedClasses.size() == 0) {
- LOGGER.warn("Unable to classify");
- return null;
- } else {
- LOGGER.debug("Classified document as " + detectedClasses);
- }
-
- // for each of these classes:
- // use TypeUtil to switch the document to that type
- Map<Class, Object> typedDocs = convertToDetectedClasses(detectedClasses, document);
-
- if (typedDocs.size() == 0) {
- LOGGER.warn("Unable to convert to any detected Class");
- return null;
- } else {
- LOGGER.debug("Document has " + typedDocs.size() + " representations: " + typedDocs.toString());
- }
-
- Map<Class, ActivityObject> convertedDocs = new HashMap<>();
-
- // for each specified / discovered converter
- for (ActivityObjectConverter converter : converters) {
-
- Class requiredClass = converter.requiredClass();
-
- Object typedDoc = typedDocs.get(requiredClass);
-
- ActivityObject activityObject = applyConverter(converter, typedDoc);
-
- convertedDocs.put(requiredClass, activityObject);
- }
-
- return deepestDescendant(convertedDocs);
- }
-
- protected ActivityObject applyConverter(ActivityObjectConverter converter, Object typedDoc) {
-
- ActivityObject activityObject = null;
- // if the document can be typed as the required class
- if (typedDoc != null) {
-
- // let the converter create activities if it can
- try {
- activityObject = convertToActivityObject(converter, typedDoc);
- } catch (Exception ex) {
- LOGGER.debug("convertToActivity caught exception " + ex.getMessage());
- }
-
- }
- return activityObject;
- }
-
- protected ActivityObject convertToActivityObject(ActivityObjectConverter converter, Object document) {
-
- ActivityObject activityObject = null;
- try {
- activityObject = converter.toActivityObject(document);
- } catch (ActivityConversionException e1) {
- LOGGER.debug(converter.getClass().getCanonicalName() + " unable to convert " + converter.requiredClass().getClass().getCanonicalName() + " to Activity");
- }
-
- return activityObject;
-
- }
-
- protected List<Class> detectClasses(Object document) {
-
- // ConcurrentHashSet is preferable, but it's only in guava 15+
- // spark 1.5.0 uses guava 14 so for the moment this is the workaround
- // Set<Class> detectedClasses = new ConcurrentHashSet();
- Set<Class> detectedClasses = Collections.newSetFromMap(new ConcurrentHashMap<Class, Boolean>());
- for (DocumentClassifier classifier : classifiers) {
- try {
- List<Class> detected = classifier.detectClasses(document);
- if (detected != null && detected.size() > 0) {
- detectedClasses.addAll(detected);
- }
- } catch (Exception ex) {
- LOGGER.warn("{} failed in method detectClasses - ()", classifier.getClass().getCanonicalName(), ex);
- }
- }
-
- return new ArrayList<>(detectedClasses);
- }
-
- private Map<Class, Object> convertToDetectedClasses(List<Class> datumClasses, Object document) {
-
- Map<Class, Object> convertedDocuments = new HashMap<>();
- for (Class detectedClass : datumClasses) {
-
- Object typedDoc;
- if (detectedClass.isInstance(document)) {
- typedDoc = document;
- } else {
- typedDoc = TypeConverterUtil.getInstance().convert(document, detectedClass);
- }
-
- if (typedDoc != null) {
- convertedDocuments.put(detectedClass, typedDoc);
- }
- }
-
- return convertedDocuments;
- }
-
- /**
- * configure ActivityObjectConverterUtil.
- */
- public void configure() {
- Reflections reflections = new Reflections(new ConfigurationBuilder()
- .setUrls(ClasspathHelper.forPackage("org.apache.streams"))
- .setScanners(new SubTypesScanner()));
- if (classifiers.size() == 0) {
- Set<Class<? extends DocumentClassifier>> classifierClasses = reflections.getSubTypesOf(DocumentClassifier.class);
- for (Class classifierClass : classifierClasses) {
- try {
- this.classifiers.add((DocumentClassifier) classifierClass.newInstance());
- } catch (Exception ex) {
- LOGGER.warn("Exception instantiating " + classifierClass);
- }
- }
- }
- Preconditions.checkArgument(classifiers.size() > 0);
- if (converters.size() == 0) {
- Set<Class<? extends ActivityObjectConverter>> converterClasses = reflections.getSubTypesOf(ActivityObjectConverter.class);
- for (Class converterClass : converterClasses) {
- try {
- this.converters.add((ActivityObjectConverter) converterClass.newInstance());
- } catch (Exception ex) {
- LOGGER.warn("Exception instantiating " + converterClass);
- }
- }
- }
- Preconditions.checkArgument(this.converters.size() > 0);
- }
-
- private boolean isAncestor(Class possibleDescendant, Class possibleAncestor) {
- return !possibleDescendant.equals(Object.class) &&
- (possibleDescendant.getSuperclass().equals(possibleAncestor) ||
- isAncestor(possibleDescendant.getSuperclass(), possibleAncestor));
- }
-
- // prefer the most specific ActivityObject sub-class returned by all converters
- private ActivityObject deepestDescendant(Map<Class, ActivityObject> map) {
-
- ActivityObject result = null;
-
- for ( Map.Entry<Class, ActivityObject> entry : map.entrySet()) {
- if ( entry.getKey() != null ) {
- if (result == null) {
- result = entry.getValue();
- } else if (isAncestor(entry.getKey(), result.getClass())) {
- result = entry.getValue();
- }
- }
- }
-
- return result;
- }
-
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseDocumentClassifier.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseDocumentClassifier.java
deleted file mode 100644
index cd99b2f..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseDocumentClassifier.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter;
-
-import org.apache.streams.data.DocumentClassifier;
-import org.apache.streams.data.util.ActivityUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Ensures generic String and ObjectNode documents can be converted to Activity
- *
- * <p/>
- * BaseDocumentClassifier is included by default in all
- * @see org.apache.streams.converter.ActivityConverterProcessor
- *
- */
-public class BaseDocumentClassifier implements DocumentClassifier {
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private static final Logger LOGGER = LoggerFactory.getLogger(BaseDocumentClassifier.class);
-
- @Override
- @SuppressWarnings("unchecked")
- public List<Class> detectClasses(Object document) {
-
- Activity activity;
- ObjectNode node = null;
-
- List<Class> classes = new ArrayList<>();
- // Soon javax.validation will available in jackson
- // That will make this simpler and more powerful
- if ( document instanceof String ) {
- classes.add(String.class);
- try {
- activity = this.mapper.readValue((String)document, Activity.class);
- if (activity != null && ActivityUtil.isValid(activity)) {
- classes.add(Activity.class);
- }
- } catch (IOException e1) {
- try {
- node = this.mapper.readValue((String)document, ObjectNode.class);
- classes.add(ObjectNode.class);
- } catch (IOException ignored) {
- LOGGER.trace("ignoring ", ignored);
- }
- }
- } else if ( document instanceof ObjectNode ) {
- classes.add(ObjectNode.class);
- activity = this.mapper.convertValue(document, Activity.class);
- if (ActivityUtil.isValid(activity)) {
- classes.add(Activity.class);
- }
- } else {
- classes.add(document.getClass());
- }
-
- return classes;
-
- }
-
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseObjectNodeActivityConverter.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseObjectNodeActivityConverter.java
deleted file mode 100644
index 6808256..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseObjectNodeActivityConverter.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter;
-
-import org.apache.streams.data.ActivityConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang.NotImplementedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Ensures generic ObjectNode representation of an Activity can be converted to Activity
- *
- * <p/>
- * BaseObjectNodeActivityConverter is included by default in all
- * @see org.apache.streams.converter.ActivityConverterProcessor
- *
- *
- */
-public class BaseObjectNodeActivityConverter implements ActivityConverter<ObjectNode> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(BaseObjectNodeActivityConverter.class);
-
- public static Class requiredClass = ObjectNode.class;
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Override
- public Class requiredClass() {
- return requiredClass;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public ObjectNode fromActivity(Activity deserialized) throws ActivityConversionException {
- try {
- return mapper.convertValue(deserialized, ObjectNode.class);
- } catch (Exception ex) {
- throw new ActivityConversionException();
- }
- }
-
- @Override
- public List<ObjectNode> fromActivityList(List<Activity> list) {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Activity> toActivityList(ObjectNode serialized) throws ActivityConversionException {
- List<Activity> activityList = new ArrayList<>();
- try {
- activityList.add(mapper.convertValue(serialized, Activity.class));
- } catch (Exception ex) {
- throw new ActivityConversionException();
- }
- return activityList;
- }
-
- @Override
- public List<Activity> toActivityList(List<ObjectNode> list) {
- List<Activity> result = new ArrayList<>();
- for ( ObjectNode item : list ) {
- try {
- result.addAll(toActivityList(item));
- } catch (ActivityConversionException ex) {
- LOGGER.trace("ActivityConversionException", ex);
- }
- }
- return result;
- }
-
-
-
-
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseObjectNodeActivityObjectConverter.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseObjectNodeActivityObjectConverter.java
deleted file mode 100644
index a38585e..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseObjectNodeActivityObjectConverter.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter;
-
-import org.apache.streams.data.ActivityObjectConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-/**
- * Ensures generic ObjectNode representation of an Activity can be converted to Activity.
- *
- * <p/>
- * BaseObjectNodeActivityConverter is included by default in all
- * @see {@link ActivityConverterProcessor}
- *
- */
-public class BaseObjectNodeActivityObjectConverter implements ActivityObjectConverter<ObjectNode> {
-
- public static Class requiredClass = ObjectNode.class;
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Override
- public Class requiredClass() {
- return requiredClass;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public ObjectNode fromActivityObject(ActivityObject deserialized) throws ActivityConversionException {
- try {
- return mapper.convertValue(deserialized, ObjectNode.class);
- } catch (Exception ex) {
- throw new ActivityConversionException();
- }
- }
-
- @Override
- public ActivityObject toActivityObject(ObjectNode serialized) throws ActivityConversionException {
- try {
- return mapper.convertValue(serialized, ActivityObject.class);
- } catch (Exception ex) {
- throw new ActivityConversionException();
- }
- }
-
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseStringActivityConverter.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseStringActivityConverter.java
deleted file mode 100644
index 74d5177..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseStringActivityConverter.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter;
-
-import org.apache.streams.data.ActivityConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang.NotImplementedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Ensures generic String Json representation of an Activity can be converted to Activity
- *
- * <p/>
- * BaseObjectNodeActivityConverter is included by default in all
- * {@link org.apache.streams.converter.ActivityConverterProcessor}
- *
- */
-public class BaseStringActivityConverter implements ActivityConverter<String> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(BaseObjectNodeActivityConverter.class);
-
- public static final Class requiredClass = String.class;
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Override
- public Class requiredClass() {
- return requiredClass;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public String fromActivity(Activity deserialized) throws ActivityConversionException {
- try {
- return mapper.writeValueAsString(deserialized);
- } catch (JsonProcessingException ex) {
- throw new ActivityConversionException();
- }
- }
-
- @Override
- public List<String> fromActivityList(List<Activity> list) {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Activity> toActivityList(String serialized) throws ActivityConversionException {
- List<Activity> activityList = new ArrayList<>();
- try {
- activityList.add(mapper.readValue(serialized, Activity.class));
- } catch (Exception ex) {
- throw new ActivityConversionException();
- }
- return activityList;
- }
-
- @Override
- public List<Activity> toActivityList(List<String> list) {
- List<Activity> result = new ArrayList<>();
- for ( String item : list ) {
- try {
- result.addAll(toActivityList(item));
- } catch (ActivityConversionException ex) {
- LOGGER.trace("ActivityConversionException", ex);
- }
- }
- return result;
- }
-
-
-
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseStringActivityObjectConverter.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseStringActivityObjectConverter.java
deleted file mode 100644
index 5434c7d..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/BaseStringActivityObjectConverter.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter;
-
-import org.apache.streams.data.ActivityObjectConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-/**
- * Ensures generic ObjectNode representation of an Activity can be converted to Activity.
- *
- * <p/>
- * BaseObjectNodeActivityConverter is included by default in all
- * {@link ActivityConverterProcessor}
- *
- */
-public class BaseStringActivityObjectConverter implements ActivityObjectConverter<String> {
-
- public static Class requiredClass = String.class;
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Override
- public Class requiredClass() {
- return requiredClass;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public String fromActivityObject(ActivityObject deserialized) throws ActivityConversionException {
- try {
- return mapper.writeValueAsString(deserialized);
- } catch (Exception ex) {
- throw new ActivityConversionException();
- }
- }
-
- @Override
- public ActivityObject toActivityObject(String serialized) throws ActivityConversionException {
- try {
- return mapper.readValue(serialized, ActivityObject.class);
- } catch (Exception ex) {
- throw new ActivityConversionException();
- }
- }
-
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/FieldConstants.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/FieldConstants.java
deleted file mode 100644
index 3443bd9..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/FieldConstants.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter;
-
-/**
- * Predefined field symbols.
- */
-public class FieldConstants {
-
- protected static final String ID = "ID";
- protected static final String SEQ = "SEQ";
- protected static final String TS = "TS";
- protected static final String META = "META";
- protected static final String DOC = "DOC";
-
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterProcessor.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterProcessor.java
deleted file mode 100644
index 41de6c5..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterProcessor.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.core.util.DatumUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * HoconConverterProcessor is a utility processor for converting any datum document
- * with translation rules expressed as HOCON in the classpath or at a URL.
- *
- * <p/>
- * To use this capability without a dedicated stream processor, just use HoconConverterUtil.
- */
-public class HoconConverterProcessor implements StreamsProcessor {
-
- public static final String STREAMS_ID = "HoconConverterProcessor";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(HoconConverterProcessor.class);
-
- protected Class outClass;
- protected String hocon;
- protected String inPath;
- protected String outPath;
-
- /**
- * HoconConverterProcessor.
- *
- * @param outClass outClass
- * @param hocon hocon
- * @param inPath inPath
- * @param outPath outPath
- */
- public HoconConverterProcessor(Class outClass, String hocon, String inPath, String outPath) {
- this.outClass = outClass;
- this.hocon = hocon;
- this.inPath = inPath;
- this.outPath = outPath;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = new LinkedList<>();
- Object document = entry.getDocument();
-
- Object outDoc = HoconConverterUtil.getInstance().convert(document, outClass, hocon, inPath, outPath);
-
- StreamsDatum datum = DatumUtils.cloneDatum(entry);
- datum.setDocument(outDoc);
- result.add(datum);
-
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
-
- }
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterUtil.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterUtil.java
deleted file mode 100644
index 1f68f6c..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterUtil.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigObject;
-import com.typesafe.config.ConfigRenderOptions;
-import com.typesafe.config.ConfigValue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * HoconConverterUtil supports HoconConverterProcessor in converting types via application
- * of hocon (https://github.com/typesafehub/config/blob/master/HOCON.md) scripts.
- */
-public class HoconConverterUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(HoconConverterUtil.class);
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private static final HoconConverterUtil INSTANCE = new HoconConverterUtil();
-
- public static HoconConverterUtil getInstance() {
- return INSTANCE;
- }
-
- public Object convert(Object object, Class outClass, String hoconResource) {
- Config hocon = ConfigFactory.parseResources(hoconResource);
- return convert(object, outClass, hocon, null);
- }
-
- public Object convert(Object object, Class outClass, String hoconResource, String outPath) {
- Config hocon = ConfigFactory.parseResources(hoconResource);
- return convert(object, outClass, hocon, outPath);
- }
-
- public Object convert(Object object, Class outClass, String hoconResource, String inPath, String outPath) {
- Config hocon = ConfigFactory.parseResources(hoconResource);
- return convert(object, outClass, hocon, inPath, outPath);
- }
-
- public Object convert(Object object, Class outClass, Config hocon, String outPath) {
- return convert(object, outClass, hocon, null, outPath);
- }
-
- /**
- * convert.
- * @param object object
- * @param outClass outClass
- * @param hocon hocon
- * @param inPath inPath
- * @param outPath outPath
- * @return result
- */
- public Object convert(Object object, Class outClass, Config hocon, String inPath, String outPath) {
- String json;
- Object outDoc = null;
- if ( object instanceof String ) {
- json = (String) object;
- } else {
- try {
- json = mapper.writeValueAsString(object);
- } catch (JsonProcessingException ex) {
- LOGGER.warn("Failed to process input:", object);
- return outDoc;
- }
- }
-
- Config base;
- if( inPath == null) {
- base = ConfigFactory.parseString(json);
- } else {
- ObjectNode node;
- try {
- node = mapper.readValue(json, ObjectNode.class);
- ObjectNode root = mapper.createObjectNode();
- root.set(inPath, node);
- json = mapper.writeValueAsString(root);
- base = ConfigFactory.parseString(json);
- } catch (Exception ex) {
- LOGGER.warn("Failed to process input:", object);
- return outDoc;
- }
- }
-
- Config converted = hocon.withFallback(base);
-
- String outJson = null;
- try {
- if( outPath == null ) {
- outJson = converted.resolve().root().render(ConfigRenderOptions.concise());
- } else {
- Config resolved = converted.resolve();
- ConfigObject outObject = resolved.withOnlyPath(outPath).root();
- ConfigValue outValue = outObject.get(outPath);
- outJson = outValue.render(ConfigRenderOptions.concise());
- }
- } catch (Exception ex) {
- LOGGER.warn("Failed to convert:", json);
- LOGGER.warn(ex.getMessage());
- }
- if ( outClass == String.class )
- return outJson;
- else {
- try {
- outDoc = mapper.readValue( outJson, outClass );
- } catch (IOException ex) {
- LOGGER.warn("Failed to convert:", object);
- }
- }
- return outDoc;
- }
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/LineReadWriteUtil.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/LineReadWriteUtil.java
deleted file mode 100644
index d680541..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/LineReadWriteUtil.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Joiner;
-import org.apache.commons.lang3.StringUtils;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * LineReadWriteUtil converts Datums to/from character array appropriate for writing to
- * file systems.
- */
-public class LineReadWriteUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TypeConverterUtil.class);
-
- private static Map<LineReadWriteConfiguration, LineReadWriteUtil> INSTANCE_MAP = new ConcurrentHashMap<>();
-
- private List<String> fields;
- private String fieldDelimiter = "\t";
- private String lineDelimiter = "\n";
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private LineReadWriteUtil() {
- }
-
- private LineReadWriteUtil(LineReadWriteConfiguration configuration) {
- this.fields = configuration.getFields();
- this.fieldDelimiter = configuration.getFieldDelimiter();
- this.lineDelimiter = configuration.getLineDelimiter();
- String encoding = configuration.getEncoding();
- }
-
- public static LineReadWriteUtil getInstance() {
- return getInstance(new LineReadWriteConfiguration());
- }
-
- /**
- * getInstance.
- * @param configuration
- * @return result
- */
- public static LineReadWriteUtil getInstance(LineReadWriteConfiguration configuration) {
- if (INSTANCE_MAP.containsKey(configuration) && INSTANCE_MAP.get(configuration) != null) {
- return INSTANCE_MAP.get(configuration);
- } else {
- INSTANCE_MAP.put(configuration, new LineReadWriteUtil(configuration));
- return INSTANCE_MAP.get(configuration);
- }
- }
-
- /**
- * processLine
- * @param line
- * @return result
- */
- public StreamsDatum processLine(String line) {
-
- List<String> expectedFields = fields;
- if ( line.endsWith(lineDelimiter)) {
- line = trimLineDelimiter(line);
- }
- String[] parsedFields = line.split(fieldDelimiter);
-
- if (parsedFields.length == 0) {
- return null;
- }
-
- String id = null;
- DateTime ts = null;
- BigInteger seq = null;
- Map<String, Object> metadata = null;
- String json = null;
-
- if ( expectedFields.contains( FieldConstants.DOC )
- && parsedFields.length > expectedFields.indexOf(FieldConstants.DOC)) {
- json = parsedFields[expectedFields.indexOf(FieldConstants.DOC)];
- }
-
- if ( expectedFields.contains( FieldConstants.ID )
- && parsedFields.length > expectedFields.indexOf(FieldConstants.ID)) {
- id = parsedFields[expectedFields.indexOf(FieldConstants.ID)];
- }
- if ( expectedFields.contains( FieldConstants.SEQ )
- && parsedFields.length > expectedFields.indexOf(FieldConstants.SEQ)) {
- try {
- seq = new BigInteger(parsedFields[expectedFields.indexOf(FieldConstants.SEQ)]);
- } catch ( NumberFormatException nfe ) {
- LOGGER.warn("invalid sequence number {}", nfe);
- }
- }
- if ( expectedFields.contains( FieldConstants.TS )
- && parsedFields.length > expectedFields.indexOf(FieldConstants.TS)) {
- ts = parseTs(parsedFields[expectedFields.indexOf(FieldConstants.TS)]);
- }
- if ( expectedFields.contains( FieldConstants.META )
- && parsedFields.length > expectedFields.indexOf(FieldConstants.META)) {
- metadata = parseMap(parsedFields[expectedFields.indexOf(FieldConstants.META)]);
- }
-
- StreamsDatum datum = new StreamsDatum(json);
- datum.setId(id);
- datum.setTimestamp(ts);
- datum.setMetadata(metadata);
- datum.setSequenceid(seq);
- return datum;
-
- }
-
- /**
- * convertResultToString
- * @param entry
- * @return result
- */
- public String convertResultToString(StreamsDatum entry) {
- String metadataJson = null;
- try {
- metadataJson = MAPPER.writeValueAsString(entry.getMetadata());
- } catch (JsonProcessingException ex) {
- LOGGER.warn("Error converting metadata to a string", ex);
- }
-
- String documentJson = null;
- try {
- if ( entry.getDocument() instanceof String ) {
- documentJson = (String) entry.getDocument();
- } else {
- documentJson = MAPPER.writeValueAsString(entry.getDocument());
- }
- } catch (JsonProcessingException ex) {
- LOGGER.warn("Error converting document to string", ex);
- }
-
- if (StringUtils.isBlank(documentJson)) {
- return null;
- } else {
- StringBuilder stringBuilder = new StringBuilder();
- Iterator<String> fields = this.fields.iterator();
- List<String> fielddata = new ArrayList<>();
- Joiner joiner = Joiner.on(fieldDelimiter).useForNull("");
- while( fields.hasNext() ) {
- String field = fields.next();
- if ( field.equals(FieldConstants.DOC) ) {
- fielddata.add(documentJson);
- } else if ( field.equals(FieldConstants.ID) ) {
- fielddata.add(entry.getId());
- } else if ( field.equals(FieldConstants.SEQ) ) {
- if (entry.getSequenceid() != null) {
- fielddata.add(entry.getSequenceid().toString());
- } else {
- fielddata.add("null");
- }
- } else if ( field.equals(FieldConstants.TS) ) {
- if (entry.getTimestamp() != null) {
- fielddata.add(entry.getTimestamp().toString());
- } else {
- fielddata.add(DateTime.now().toString());
- }
- } else if ( field.equals(FieldConstants.META) ) {
- fielddata.add(metadataJson);
- } else if ( entry.getMetadata().containsKey(field)) {
- fielddata.add(entry.getMetadata().get(field).toString());
- } else {
- fielddata.add(null);
- }
- }
- joiner.appendTo(stringBuilder, fielddata);
- return stringBuilder.toString();
- }
- }
-
- /**
- * parseTs
- * @param field - dateTime string to be parsed
- * @return {@link DateTime}
- */
- public DateTime parseTs(String field) {
-
- DateTime timestamp = null;
- try {
- long longts = Long.parseLong(field);
- timestamp = new DateTime(longts);
- } catch ( Exception e1 ) {
- try {
- timestamp = DateTime.parse(field);
- } catch ( Exception e2 ) {
- try {
- timestamp = MAPPER.readValue(field, DateTime.class);
- } catch ( Exception e3 ) {
- LOGGER.warn("Could not parse timestamp:{} ", field);
- }
- }
- }
-
- return timestamp;
- }
-
- /**
- * parseMap
- * @param field
- * @return result
- */
- public Map<String, Object> parseMap(String field) {
-
- Map<String, Object> metadata = null;
-
- try {
- JsonNode jsonNode = MAPPER.readValue(field, JsonNode.class);
- metadata = MAPPER.convertValue(jsonNode, Map.class);
- } catch (Exception ex) {
- LOGGER.warn("failed in parseMap: " + ex.getMessage());
- }
- return metadata;
- }
-
- private String trimLineDelimiter(String str) {
- if (!StringUtils.isNotBlank(str)) {
- if (str.endsWith(lineDelimiter)) {
- return str.substring(0, str.length() - 1);
- }
- }
- return str;
- }
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/TypeConverterProcessor.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/TypeConverterProcessor.java
deleted file mode 100644
index bd0b304..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/TypeConverterProcessor.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * TypeConverterProcessor converts between String json and jackson-compatible POJO objects.
- *
- * <p/>
- * Activity is one supported jackson-compatible POJO, so JSON String and objects with structual similarities
- * to Activity can be converted to Activity objects.
- *
- * <p/>
- * However, conversion to Activity should probably use {@link org.apache.streams.converter.ActivityConverterProcessor}
- *
- */
-public class TypeConverterProcessor implements StreamsProcessor, Serializable {
-
- public static final String STREAMS_ID = "TypeConverterProcessor";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TypeConverterProcessor.class);
-
- private List<String> formats = new ArrayList<>();
-
- protected ObjectMapper mapper;
-
- protected Class outClass;
-
- public TypeConverterProcessor(Class outClass) {
- this.outClass = outClass;
- }
-
- public TypeConverterProcessor(Class outClass, List<String> formats) {
- this(outClass);
- this.formats = formats;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = new LinkedList<>();
- Object inDoc = entry.getDocument();
-
- Object outDoc = TypeConverterUtil.getInstance().convert(inDoc, outClass, mapper);
-
- if ( outDoc != null ) {
- entry.setDocument(outDoc);
- result.add(entry);
- }
-
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- if ( formats.size() > 0 ) {
- this.mapper = StreamsJacksonMapper.getInstance(formats);
- } else {
- this.mapper = StreamsJacksonMapper.getInstance();
- }
- }
-
- @Override
- public void cleanUp() {
- this.mapper = null;
- }
-
-}
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/TypeConverterUtil.java b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/TypeConverterUtil.java
deleted file mode 100644
index abf3151..0000000
--- a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/TypeConverterUtil.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * TypeConverterUtil supports TypeConverterProcessor in converting between String json and
- * jackson-compatible POJO objects.
- */
-public class TypeConverterUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TypeConverterUtil.class);
-
- private static final TypeConverterUtil INSTANCE = new TypeConverterUtil();
-
- public static TypeConverterUtil getInstance() {
- return INSTANCE;
- }
-
- public Object convert(Object object, Class outClass) {
- return TypeConverterUtil.getInstance().convert(object, outClass, StreamsJacksonMapper.getInstance());
- }
-
- /**
- * convert
- * @param object
- * @param outClass
- * @param mapper
- * @return
- */
- public Object convert(Object object, Class outClass, ObjectMapper mapper) {
- ObjectNode node = null;
- Object outDoc = null;
- if ( object instanceof String ) {
- try {
- node = mapper.readValue((String)object, ObjectNode.class);
- } catch (IOException ex) {
- LOGGER.warn(ex.getMessage());
- LOGGER.warn(object.toString());
- }
- } else {
- node = mapper.convertValue(object, ObjectNode.class);
- }
-
- if(node != null) {
- try {
- if ( outClass == String.class ) {
- outDoc = mapper.writeValueAsString(node);
- } else {
- outDoc = mapper.convertValue(node, outClass);
- }
- } catch (Throwable ex) {
- LOGGER.warn(ex.getMessage());
- LOGGER.warn(node.toString());
- }
- }
-
- return outDoc;
- }
-}
diff --git a/streams-components/streams-converters/src/main/jsonschema/org/apache/streams/converter/ActivityConverterProcessorConfiguration.json b/streams-components/streams-converters/src/main/jsonschema/org/apache/streams/converter/ActivityConverterProcessorConfiguration.json
deleted file mode 100644
index f535b5a..0000000
--- a/streams-components/streams-converters/src/main/jsonschema/org/apache/streams/converter/ActivityConverterProcessorConfiguration.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.converter.ActivityConverterProcessorConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "classifiers": {
- "type": "array",
- "items": {
- "javaType": "org.apache.streams.data.DocumentClassifier",
- "type": "object"
- }
- },
- "converters": {
- "type": "array",
- "items": {
- "javaType": "org.apache.streams.data.ActivityConverter",
- "type": "object"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-components/streams-converters/src/main/jsonschema/org/apache/streams/converter/ActivityObjectConverterProcessorConfiguration.json b/streams-components/streams-converters/src/main/jsonschema/org/apache/streams/converter/ActivityObjectConverterProcessorConfiguration.json
deleted file mode 100644
index 7eacacb..0000000
--- a/streams-components/streams-converters/src/main/jsonschema/org/apache/streams/converter/ActivityObjectConverterProcessorConfiguration.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.converter.ActivityObjectConverterProcessorConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "classifiers": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "org.apache.streams.data.DocumentClassifier"
- }
- },
- "converters": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "org.apache.streams.data.ActivityObjectConverter"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-components/streams-converters/src/main/jsonschema/org/apache/streams/converter/LineReadWriteConfiguration.json b/streams-components/streams-converters/src/main/jsonschema/org/apache/streams/converter/LineReadWriteConfiguration.json
deleted file mode 100644
index 13e8428..0000000
--- a/streams-components/streams-converters/src/main/jsonschema/org/apache/streams/converter/LineReadWriteConfiguration.json
+++ /dev/null
@@ -1,36 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.converter.LineReadWriteConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "fields": {
- "type": "array",
- "items": {
- "type": "string"
- },
- "default": [
- "ID",
- "TS",
- "META",
- "DOC"
- ]
- },
- "field_delimiter": {
- "type": "string",
- "default": "\t"
- },
- "line_delimiter": {
- "type": "string",
- "default": "\n"
- },
- "encoding": {
- "type": "string",
- "default": "UTF-8"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-components/streams-converters/src/site/markdown/index.md b/streams-components/streams-converters/src/site/markdown/index.md
deleted file mode 100644
index dde0190..0000000
--- a/streams-components/streams-converters/src/site/markdown/index.md
+++ /dev/null
@@ -1,8 +0,0 @@
-streams-converters
-=====================
-
-Apply a filter to an activity stream, dropping any document that does not match the criteria
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/BaseActivityConverterProcessorTest.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/BaseActivityConverterProcessorTest.java
deleted file mode 100644
index 2b41459..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/BaseActivityConverterProcessorTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter.test;
-
-import org.apache.streams.converter.ActivityConverterProcessor;
-import org.apache.streams.converter.ActivityConverterProcessorConfiguration;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for
- * {@link org.apache.streams.converter.ActivityConverterProcessor}
- *
- * Test that default String & ObjectNode conversion works.
- */
-public class BaseActivityConverterProcessorTest {
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private static final String ACTIVITY_JSON = "{\"id\":\"id\",\"published\":\"Tue Jan 17 21:21:46 Z 2012\",\"verb\":\"post\",\"provider\":{\"id\":\"providerid\"}}";
-
- private ActivityConverterProcessor processor;
-
- @Before
- public void setup() {
- processor = new ActivityConverterProcessor(new ActivityConverterProcessorConfiguration());
- processor.prepare(new ActivityConverterProcessorConfiguration());
- }
-
- @Test
- public void testBaseActivitySerializerProcessorInvalid() {
- String INVALID_DOCUMENT = " 38Xs}";
- StreamsDatum datum = new StreamsDatum(INVALID_DOCUMENT);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(0, result.size());
- }
-
- @Test
- public void testActivityConverterProcessorString() {
- StreamsDatum datum = new StreamsDatum(ACTIVITY_JSON);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof Activity);
- assertTrue(((Activity)resultDatum.getDocument()).getVerb().equals("post"));
- }
-
- @Test
- public void testBaseActivitySerializerProcessorObject() throws IOException {
- ObjectNode OBJECT_DOCUMENT = mapper.readValue(ACTIVITY_JSON, ObjectNode.class);
- StreamsDatum datum = new StreamsDatum(OBJECT_DOCUMENT);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof Activity);
- assertTrue(((Activity)resultDatum.getDocument()).getVerb().equals("post"));
- }
-
-}
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/BaseActivityObjectConverterProcessorTest.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/BaseActivityObjectConverterProcessorTest.java
deleted file mode 100644
index ac78b0e..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/BaseActivityObjectConverterProcessorTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter.test;
-
-import org.apache.streams.converter.ActivityObjectConverterProcessor;
-import org.apache.streams.converter.ActivityObjectConverterProcessorConfiguration;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for
- * {@link ActivityObjectConverterProcessor}
- *
- * Test that default String & ObjectNode conversion works.
- */
-public class BaseActivityObjectConverterProcessorTest {
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private static final String ACTIVITYOBJECT_JSON = "{\"id\":\"id\",\"objectType\":\"person\"}";
-
- private ActivityObjectConverterProcessor processor;
-
- @Before
- public void setup() {
- processor = new ActivityObjectConverterProcessor(new ActivityObjectConverterProcessorConfiguration());
- processor.prepare(new ActivityObjectConverterProcessorConfiguration());
- }
-
- @Test
- public void testBaseActivityObjectSerializerProcessorInvalid() {
- String INVALID_DOCUMENT = " 38Xs}";
- StreamsDatum datum = new StreamsDatum(INVALID_DOCUMENT);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(0, result.size());
- }
-
- @Test
- public void testActivityObjectConverterProcessorString() {
- StreamsDatum datum = new StreamsDatum(ACTIVITYOBJECT_JSON);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof ActivityObject);
- assertTrue(((ActivityObject)resultDatum.getDocument()).getObjectType().equals("person"));
- }
-
- @Test
- public void testBaseActivityObjectSerializerProcessorObject() throws IOException {
- ObjectNode OBJECT_DOCUMENT = mapper.readValue(ACTIVITYOBJECT_JSON, ObjectNode.class);
- StreamsDatum datum = new StreamsDatum(OBJECT_DOCUMENT);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof ActivityObject);
- assertTrue(((ActivityObject)resultDatum.getDocument()).getObjectType().equals("person"));
- }
-
-}
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomActivityConverter.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomActivityConverter.java
deleted file mode 100644
index 8d55e89..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomActivityConverter.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter.test;
-
-import org.apache.streams.data.ActivityConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Provider;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Support class for
- * {@link org.apache.streams.converter.test.CustomActivityConverterProcessorTest}
- */
-public class CustomActivityConverter implements ActivityConverter<CustomType> {
-
-
- @Override
- public Class requiredClass() {
- return CustomType.class;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public CustomType fromActivity(Activity deserialized) throws ActivityConversionException {
- return null;
- }
-
- @Override
- public List<Activity> toActivityList(CustomType document) throws ActivityConversionException {
- Activity customActivity = new Activity();
- customActivity.setId(document.getTest());
- customActivity.setVerb(document.getTest());
- customActivity.setProvider((Provider)new Provider().withId(document.getTest()));
- List<Activity> activityList = new ArrayList<>();
- activityList.add(customActivity);
- return activityList;
- }
-
- @Override
- public List<Activity> toActivityList(List<CustomType> list) throws ActivityConversionException {
- return null;
- }
-
- @Override
- public List<CustomType> fromActivityList(List<Activity> list) throws ActivityConversionException {
- return null;
- }
-}
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomActivityConverterProcessorTest.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomActivityConverterProcessorTest.java
deleted file mode 100644
index d5ef633..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomActivityConverterProcessorTest.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter.test;
-
-import org.apache.streams.converter.ActivityConverterProcessor;
-import org.apache.streams.converter.ActivityConverterProcessorConfiguration;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for
- * {@link org.apache.streams.converter.ActivityConverterProcessor}
- *
- * Test that arbitrary POJO conversion works, including when POJO represented as String & ObjectNode.
- */
-public class CustomActivityConverterProcessorTest {
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private ActivityConverterProcessor processor;
-
- private CustomType testDocument;
-
- @Before
- public void setup() {
- ActivityConverterProcessorConfiguration configuration = new ActivityConverterProcessorConfiguration();
- configuration.getClassifiers().add(new CustomDocumentClassifier());
- configuration.getConverters().add(new CustomActivityConverter());
- processor = new ActivityConverterProcessor(configuration);
- processor.prepare(configuration);
- testDocument = new CustomType();
- testDocument.setTest("testValue");
- }
-
- @Test
- public void testCustomActivityConverterProcessorString() throws IOException {
- StreamsDatum datum = new StreamsDatum(mapper.writeValueAsString(testDocument));
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof Activity);
- assertTrue(((Activity)resultDatum.getDocument()).getVerb().equals("testValue"));
- }
-
- @Test
- public void testCustomActivitySerializerProcessorObjectNode() throws IOException {
- ObjectNode OBJECT_DOCUMENT = mapper.convertValue(testDocument, ObjectNode.class);
- StreamsDatum datum = new StreamsDatum(OBJECT_DOCUMENT);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof Activity);
- assertTrue(((Activity)resultDatum.getDocument()).getVerb().equals("testValue"));
- }
-
- @Test
- public void testCustomActivitySerializerProcessorPOJO() throws IOException {
- StreamsDatum datum = new StreamsDatum(testDocument);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof Activity);
- assertTrue(((Activity)resultDatum.getDocument()).getVerb().equals("testValue"));
- }
-
-}
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomActivityObjectConverter.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomActivityObjectConverter.java
deleted file mode 100644
index 8296a99..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomActivityObjectConverter.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter.test;
-
-import org.apache.streams.data.ActivityObjectConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.ActivityObject;
-
-/**
- * Support class for
- * {@link CustomActivityObjectConverterProcessorTest}
- */
-public class CustomActivityObjectConverter implements ActivityObjectConverter<CustomType> {
-
- @Override
- public Class requiredClass() {
- return CustomType.class;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public CustomType fromActivityObject(ActivityObject deserialized) throws ActivityConversionException {
- CustomType customType = new CustomType();
- customType.setTest(deserialized.getId());
- return customType;
- }
-
- @Override
- public ActivityObject toActivityObject(CustomType document) throws ActivityConversionException {
- ActivityObject customActivityObject = new ActivityObject();
- customActivityObject.setId(document.getTest());
- customActivityObject.setObjectType(document.getTest());
- return customActivityObject;
- }
-
-}
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomActivityObjectConverterProcessorTest.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomActivityObjectConverterProcessorTest.java
deleted file mode 100644
index 4cf2907..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomActivityObjectConverterProcessorTest.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter.test;
-
-import org.apache.streams.converter.ActivityObjectConverterProcessor;
-import org.apache.streams.converter.ActivityObjectConverterProcessorConfiguration;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for {@link ActivityObjectConverterProcessor}
- *
- * Test that arbitrary POJO conversion works, including when POJO represented as String & ObjectNode.
- */
-public class CustomActivityObjectConverterProcessorTest {
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private ActivityObjectConverterProcessor processor;
-
- private CustomType testDocument;
-
- @Before
- public void setup() {
- ActivityObjectConverterProcessorConfiguration configuration = new ActivityObjectConverterProcessorConfiguration();
- configuration.getClassifiers().add(new CustomDocumentClassifier());
- configuration.getConverters().add(new CustomActivityObjectConverter());
- processor = new ActivityObjectConverterProcessor(configuration);
- processor.prepare(configuration);
- testDocument = new CustomType();
- testDocument.setTest("testValue");
- }
-
- @Test
- public void testCustomActivityObjectConverterProcessorString() throws IOException {
- StreamsDatum datum = new StreamsDatum(mapper.writeValueAsString(testDocument));
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof ActivityObject);
- assertTrue(((ActivityObject)resultDatum.getDocument()).getObjectType().equals("testValue"));
- }
-
- @Test
- public void testCustomActivityObjectSerializerProcessorObjectNode() throws IOException {
- ObjectNode OBJECT_DOCUMENT = mapper.convertValue(testDocument, ObjectNode.class);
- StreamsDatum datum = new StreamsDatum(OBJECT_DOCUMENT);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof ActivityObject);
- assertTrue(((ActivityObject)resultDatum.getDocument()).getObjectType().equals("testValue"));
- }
-
- @Test
- public void testCustomActivityObjectSerializerProcessorPOJO() throws IOException {
- StreamsDatum datum = new StreamsDatum(testDocument);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof ActivityObject);
- assertTrue(((ActivityObject)resultDatum.getDocument()).getObjectType().equals("testValue"));
- }
-
-}
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomDocumentClassifier.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomDocumentClassifier.java
deleted file mode 100644
index 9f8f8a9..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomDocumentClassifier.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter.test;
-
-import org.apache.streams.data.DocumentClassifier;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Support class for
- * {@link org.apache.streams.converter.test.CustomActivityConverterProcessorTest}
- */
-public class CustomDocumentClassifier implements DocumentClassifier {
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Override
- public List<Class> detectClasses(Object document) {
-
- CustomType possibleMatchDocument;
- ObjectNode node = null;
-
- List<Class> classes = new ArrayList<>();
-
- if( document instanceof String ) {
- classes.add(String.class);
- try {
- possibleMatchDocument = this.mapper.readValue((String)document, CustomType.class);
- if(possibleMatchDocument != null && possibleMatchDocument.getTest() != null)
- classes.add(CustomType.class);
- } catch (IOException e1) {
- try {
- node = this.mapper.readValue((String)document, ObjectNode.class);
- classes.add(ObjectNode.class);
- } catch (IOException ignored) { }
- }
- } else if( document instanceof ObjectNode ){
- classes.add(ObjectNode.class);
- possibleMatchDocument = this.mapper.convertValue(document, CustomType.class);
- if(possibleMatchDocument != null && possibleMatchDocument.getTest() != null) {
- classes.add(CustomType.class);
- }
- } else {
- classes.add(document.getClass());
- }
-
- return classes;
- }
-}
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomType.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomType.java
deleted file mode 100644
index 70d2391..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/CustomType.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.converter.test;
-
-/**
- * Support class for
- * {@link org.apache.streams.converter.test.CustomActivityConverterProcessorTest}
- * {@link org.apache.streams.converter.test.CustomActivityObjectConverterProcessorTest}
- */
-public class CustomType {
-
- private String test;
-
- public String getTest() {
- return test;
- }
-
- public void setTest(String test) {
- this.test = test;
- }
-}
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterProcessorTest.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterProcessorTest.java
deleted file mode 100644
index 081e29a..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterProcessorTest.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter.test;
-
-import org.apache.streams.converter.HoconConverterProcessor;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.junit.Test;
-
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests for {@link HoconConverterProcessor}
- */
-public class HoconConverterProcessorTest {
-
- /**
- * Tests in-place simple conversion from String to String
- */
- @Test
- public void testHoconConverter1() {
-
- final String TEST_JSON_1 = "{\"race\":\"klingon\",\"gender\":\"male\"}";
-
- StreamsProcessor processor = new HoconConverterProcessor(String.class, "test1.conf", null, null);
- processor.prepare(null);
- StreamsDatum datum = new StreamsDatum(TEST_JSON_1, "1");
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertTrue(resultDatum.getDocument() instanceof String);
- String result1 = (String) resultDatum.getDocument();
- assertTrue(result1.contains("race"));
- assertTrue(result1.contains("18"));
- assertTrue(result1.contains("female"));
- }
-
- /**
- * Tests derived object substitution conversion from String to ObjectNode
- */
- @Test
- public void testHoconConverter2() {
-
- final String TEST_ID_2 = "2";
- final String TEST_JSON_2 = "{\"race\":\"klingon\",\"gender\":\"male\",\"age\":18}";
-
- StreamsProcessor processor = new HoconConverterProcessor(ObjectNode.class, "test2.conf", null, "demographics");
- processor.prepare(null);
- StreamsDatum datum = new StreamsDatum(TEST_JSON_2, TEST_ID_2);
- List<StreamsDatum> result2 = processor.process(datum);
- assertNotNull(result2);
- assertEquals(1, result2.size());
- StreamsDatum resultDatum = result2.get(0);
- assertTrue(resultDatum.getDocument() instanceof ObjectNode);
- assertTrue(resultDatum.getId().equals(TEST_ID_2));
- ObjectNode resultDoc = (ObjectNode) resultDatum.getDocument();
- assertNotNull(resultDoc);
- assertTrue(resultDoc.get("race") != null);
- assertTrue(resultDoc.get("age").asDouble() == 18);
- assertTrue(resultDoc.get("gender").asText().equals("female"));
- }
-
- /**
- * Tests derived object import conversion from String to Activity
- */
- @Test
- public void testHoconConverter3() {
-
- final String TEST_JSON_3 = "{\"id\":\"123\",\"text\":\"buncha stuff\",\"user\":{\"name\":\"guy\"}}";
-
- StreamsProcessor processor = new HoconConverterProcessor(Activity.class, "test3a.conf", null, "activity");
- processor.prepare(null);
- StreamsDatum datum = new StreamsDatum(TEST_JSON_3, "3");
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertTrue(resultDatum.getDocument() instanceof Activity);
- Activity result3 = (Activity) resultDatum.getDocument();
- assertNotNull(result3);
- assertTrue(result3.getProvider() != null);
- assertTrue(result3.getId().equals("id:123"));
- assertTrue(result3.getContent().endsWith("stuff"));
- assertTrue(result3.getActor().getDisplayName().equals("Jorge"));
-
- }
-
-}
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterTest.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterTest.java
deleted file mode 100644
index 58ccad6..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter.test;
-
-import org.apache.streams.converter.HoconConverterUtil;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.junit.Test;
-
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests for {@link HoconConverterUtil}
- */
-public class HoconConverterTest {
-
- /**
- * Tests in-place simple conversion from String to String
- */
- @Test
- public void testHoconConverter1() {
-
- final String TEST_JSON_1 = "{\"race\":\"klingon\",\"gender\":\"male\"}";
- String result1 = (String) HoconConverterUtil.getInstance().convert(TEST_JSON_1, String.class, "test1.conf");
- assertNotNull(result1);
- assertTrue(result1.contains("race"));
- assertTrue(result1.contains("18"));
- assertTrue(result1.contains("female"));
- }
-
- /**
- * Tests derived object substitution conversion from String to ObjectNode
- */
- @Test
- public void testHoconConverter2() {
-
- final String TEST_JSON_2 = "{\"race\":\"klingon\",\"gender\":\"male\",\"age\":18}";
- ObjectNode result2 = (ObjectNode) HoconConverterUtil.getInstance().convert(TEST_JSON_2, ObjectNode.class, "test2.conf", "demographics");
- assertNotNull(result2);
- assertTrue(result2.get("race") != null);
- assertTrue(result2.get("age").asDouble() == 18);
- assertTrue(result2.get("gender").asText().equals("female"));
- }
-
- /**
- * Tests derived object import conversion from String to Activity
- */
- @Test
- public void testHoconConverter3() {
-
- final String TEST_JSON_3 = "{\"id\":\"123\",\"text\":\"buncha stuff\",\"user\":{\"name\":\"guy\"}}";
- Activity result3 = (Activity) HoconConverterUtil.getInstance().convert(TEST_JSON_3, Activity.class, "test3a.conf", null, "activity");
- assertNotNull(result3);
- assertTrue(result3.getProvider() != null);
- assertTrue(result3.getId().equals("id:123"));
- assertTrue(result3.getContent().endsWith("stuff"));
- assertTrue(result3.getActor().getDisplayName().equals("Jorge"));
-
- }
-
- /**
- * Tests derived object import conversion from String to Activity
- */
- @Test
- public void testHoconConverter4() {
-
- final String TEST_JSON_4 = "{\"id\":\"123\",\"name\":\"nahme\",\"screenName\":\"screeny\",\"summary\":\"sumar\"}";
- String result4 = (String) HoconConverterUtil.getInstance().convert(TEST_JSON_4, String.class, "test4.conf", "actor", "profile");
- assertNotNull(result4);
- assertTrue(result4.contains("123"));
- assertTrue(result4.contains("\"nahme\""));
- assertTrue(result4.contains("\"screeny\""));
- assertTrue(result4.contains("\"sumar\""));
-
- }
-
-}
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/NoConfigActivityConverterProcessorTest.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/NoConfigActivityConverterProcessorTest.java
deleted file mode 100644
index e0a19d1..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/NoConfigActivityConverterProcessorTest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter.test;
-
-import org.apache.streams.converter.ActivityConverterProcessor;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for
- * {@link org.apache.streams.converter.ActivityConverterProcessor}
- *
- * Test that default String & ObjectNode conversion works.
- */
-public class NoConfigActivityConverterProcessorTest {
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private static final String ACTIVITY_JSON = "{\"id\":\"id\",\"published\":\"Tue Jan 17 21:21:46 Z 2012\",\"verb\":\"post\",\"provider\":{\"id\":\"providerid\"}}";
-
- private ActivityConverterProcessor processor;
-
- @Before
- public void setup() {
- processor = new ActivityConverterProcessor();
- processor.prepare(null);
- }
-
- @Test
- public void testNoConfigActivitySerializerProcessorInvalid() {
- String INVALID_DOCUMENT = " 38Xs}";
- StreamsDatum datum = new StreamsDatum(INVALID_DOCUMENT);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(0, result.size());
- }
-
- @Test
- public void testNoConfigActivityConverterProcessorString() {
- StreamsDatum datum = new StreamsDatum(ACTIVITY_JSON);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof Activity);
- assertTrue(((Activity)resultDatum.getDocument()).getVerb().equals("post"));
- }
-
- @Test
- public void testNoConfigActivitySerializerProcessorObject() throws IOException {
- ObjectNode OBJECT_DOCUMENT = mapper.readValue(ACTIVITY_JSON, ObjectNode.class);
- StreamsDatum datum = new StreamsDatum(OBJECT_DOCUMENT);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof Activity);
- assertTrue(((Activity)resultDatum.getDocument()).getVerb().equals("post"));
- }
-
-}
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/TestLineReadWriteUtil.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/TestLineReadWriteUtil.java
deleted file mode 100644
index 635d85a..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/TestLineReadWriteUtil.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter.test;
-
-import org.apache.streams.converter.LineReadWriteConfiguration;
-import org.apache.streams.converter.LineReadWriteUtil;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang3.StringUtils;
-import org.joda.time.DateTime;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-
-/**
- * Tests for {@link LineReadWriteUtil}
- */
-public class TestLineReadWriteUtil {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(TestLineReadWriteUtil.class);
-
- private ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private static Random rand = new Random();
-
- @Test
- public void TestLineReadWrite () throws Exception {
-
- List<LineReadWriteConfiguration> configs = new ArrayList<>();
- configs.add(new LineReadWriteConfiguration());
- configs.add(new LineReadWriteConfiguration()
- .withFields(Collections.singletonList("ID")));
- configs.add(new LineReadWriteConfiguration()
- .withFields(Collections.singletonList("DOC"))
- .withFieldDelimiter("\t"));
- configs.add(new LineReadWriteConfiguration()
- .withFields(Arrays.asList("ID", "DOC"))
- .withFieldDelimiter("\t")
- .withLineDelimiter("\n"));
- configs.add(new LineReadWriteConfiguration()
- .withFields(Arrays.asList("ID", "TS", "DOC"))
- .withLineDelimiter("\n"));
- configs.add(new LineReadWriteConfiguration()
- .withFields(Arrays.asList("ID", "TS", "META", "DOC"))
- .withFieldDelimiter("|")
- .withLineDelimiter("\n"));
- configs.add(new LineReadWriteConfiguration()
- .withFields(Arrays.asList("ID", "SEQ", "TS", "META", "DOC"))
- .withFieldDelimiter("|")
- .withLineDelimiter("\\0"));
-
- for(LineReadWriteConfiguration config : configs)
- TestLineReadWriteCase(config);
-
- }
-
- public void TestLineReadWriteCase(LineReadWriteConfiguration lineReadWriteConfiguration) throws Exception {
-
- LineReadWriteUtil lineReadWriteUtil;
-
- lineReadWriteUtil = LineReadWriteUtil.getInstance(lineReadWriteConfiguration);
-
- assert(lineReadWriteUtil != null);
- StreamsDatum testDatum = randomDatum();
- String writeResult = lineReadWriteUtil.convertResultToString(testDatum);
- assert StringUtils.isNotBlank(writeResult);
- StreamsDatum readResult = lineReadWriteUtil.processLine(writeResult);
- assert readResult != null;
- assert StringUtils.isNotBlank(readResult.getId()) || StringUtils.isNotBlank((String)readResult.getDocument());
-
- }
-
- public static StreamsDatum randomDatum() {
-
- StreamsDatum datum = new StreamsDatum(UUID.randomUUID().toString());
- datum.setId(UUID.randomUUID().toString());
- datum.setTimestamp(DateTime.now());
- BigInteger result = new BigInteger(64, rand);
- datum.setSequenceid(result);
- Map<String,Object> metadata = new HashMap<>();
- metadata.put("a", UUID.randomUUID().toString());
- datum.setMetadata(metadata);
- return datum;
- }
-}
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/TypeConverterProcessorTest.java b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/TypeConverterProcessorTest.java
deleted file mode 100644
index e692444..0000000
--- a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/TypeConverterProcessorTest.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.converter.test;
-
-import org.apache.streams.converter.TypeConverterProcessor;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.data.util.ActivityUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-
-/**
- * Tests for
- * @see org.apache.streams.converter.TypeConverterProcessor
- */
-public class TypeConverterProcessorTest {
-
- private static final String DATASIFT_JSON = "{\"demographic\":{\"gender\":\"female\"},\"interaction\":{\"schema\":{\"version\":3},\"source\":\"Twitter for Android\",\"author\":{\"username\":\"ViiOLeee\",\"name\":\"Violeta Anguita\",\"id\":70931384,\"avatar\":\"http://pbs.twimg.com/profile_images/378800000851401229/bbf480cde2e9923a1d20acd393da0212_normal.jpeg\",\"link\":\"http://twitter.com/ViiOLeee\",\"language\":\"en\"},\"type\":\"twitter\",\"created_at\":\"Tue, 27 May 2014 22:38:15 [...]
-
- private static final String ACTIVITY_JSON = "{\"id\":\"id\",\"published\":\"Tue Jan 17 21:21:46 Z 2012\",\"verb\":\"post\",\"provider\":{\"id\":\"providerid\"}}";
-
- public static final String DATASIFT_FORMAT = "EEE, dd MMM yyyy HH:mm:ss Z";
-
- /**
- * Tests conversion from String to String
- */
- @Test
- public void testTypeConverterStringToString() {
- final String ID = "1";
- StreamsProcessor processor = new TypeConverterProcessor(String.class, Collections.singletonList(DATASIFT_FORMAT));
- processor.prepare(null);
- StreamsDatum datum = new StreamsDatum(DATASIFT_JSON, ID);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof String);
- assertEquals(ID, resultDatum.getId());
- }
-
- /**
- * Tests conversion from String to ObjectNode
- */
- @Test
- public void testTypeConverterStringToObjectNode() {
- final String ID = "1";
- StreamsProcessor processor = new TypeConverterProcessor(ObjectNode.class, Collections.singletonList(DATASIFT_FORMAT));
- processor.prepare(null);
- StreamsDatum datum = new StreamsDatum(DATASIFT_JSON, ID);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof ObjectNode);
- assertEquals(ID, resultDatum.getId());
- }
-
- /**
- * Tests conversion from ObjectNode to String
- */
- @Test
- public void testTypeConverterObjectNodeToString() throws IOException {
- final String ID = "1";
- StreamsProcessor processor = new TypeConverterProcessor(String.class, Collections.singletonList(DATASIFT_FORMAT));
- processor.prepare(null);
- ObjectMapper mapper = StreamsJacksonMapper.getInstance(DATASIFT_FORMAT);
- ObjectNode node = mapper.readValue(DATASIFT_JSON, ObjectNode.class);
- StreamsDatum datum = new StreamsDatum(node, ID);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof String);
- assertEquals(ID, resultDatum.getId());
- }
-
- /**
- * Tests conversion from String to POJO (Activity)
- */
- @Test
- public void testTypeConverterStringToActivity() {
- final String ID = "1";
- StreamsProcessor processor = new TypeConverterProcessor(Activity.class, Collections.singletonList(DATASIFT_FORMAT));
- processor.prepare(null);
- StreamsDatum datum = new StreamsDatum(ACTIVITY_JSON, ID);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof Activity);
- assertTrue(ActivityUtil.isValid((Activity)resultDatum.getDocument()));
- assertEquals(ID, resultDatum.getId());
- }
-
- /**
- * Tests conversion from ObjectNode to POJO (Activity)
- */
- @Test
- public void testTypeConverterObjectNodeToActivity() throws IOException {
- final String ID = "1";
- StreamsProcessor processor = new TypeConverterProcessor(Activity.class, Collections.singletonList(DATASIFT_FORMAT));
- processor.prepare(null);
- ObjectMapper mapper = StreamsJacksonMapper.getInstance(DATASIFT_FORMAT);
- ObjectNode node = mapper.readValue(ACTIVITY_JSON, ObjectNode.class);
- StreamsDatum datum = new StreamsDatum(node, ID);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof Activity);
- assertTrue(ActivityUtil.isValid((Activity)resultDatum.getDocument()));
- assertEquals(ID, resultDatum.getId());
- }
-
- /**
- * Tests conversion from POJO (Activity) to String
- */
- @Test
- public void testTypeConverterActivityToString() throws IOException {
- final String ID = "1";
- StreamsProcessor processor = new TypeConverterProcessor(String.class, Collections.singletonList(DATASIFT_FORMAT));
- processor.prepare(null);
- ObjectMapper mapper = StreamsJacksonMapper.getInstance(DATASIFT_FORMAT);
- Activity node = mapper.readValue(ACTIVITY_JSON, Activity.class);
- StreamsDatum datum = new StreamsDatum(node, ID);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof String);
- assertEquals(ID, resultDatum.getId());
- }
-
- @Test
- public void testTypeConverterActivityToObjectNode() throws IOException {
- final String ID = "1";
- StreamsProcessor processor = new TypeConverterProcessor(ObjectNode.class, Collections.singletonList(DATASIFT_FORMAT));
- processor.prepare(null);
- ObjectMapper mapper = StreamsJacksonMapper.getInstance(DATASIFT_FORMAT);
- Activity node = mapper.readValue(ACTIVITY_JSON, Activity.class);
- StreamsDatum datum = new StreamsDatum(node, ID);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof ObjectNode);
- assertEquals(ID, resultDatum.getId());
- }
-
-
-}
diff --git a/streams-components/streams-converters/src/test/resources/test1.conf b/streams-components/streams-converters/src/test/resources/test1.conf
deleted file mode 100644
index 2fa3a5a..0000000
--- a/streams-components/streams-converters/src/test/resources/test1.conf
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-age = 18
-gender = female
diff --git a/streams-components/streams-converters/src/test/resources/test2.conf b/streams-components/streams-converters/src/test/resources/test2.conf
deleted file mode 100644
index bfbf312..0000000
--- a/streams-components/streams-converters/src/test/resources/test2.conf
+++ /dev/null
@@ -1,19 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-demographics.age = 17
-demographics.age = ${?age}
-demographics.race = ${race}
-demographics.gender = fe${gender}
diff --git a/streams-components/streams-converters/src/test/resources/test3a.conf b/streams-components/streams-converters/src/test/resources/test3a.conf
deleted file mode 100644
index b63a0aa..0000000
--- a/streams-components/streams-converters/src/test/resources/test3a.conf
+++ /dev/null
@@ -1,19 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-activity.provider.id = test
-activity.id = "id:"${id}
-activity.content = ${text}
-activity.actor = {include "test3b.conf"}
diff --git a/streams-components/streams-converters/src/test/resources/test3b.conf b/streams-components/streams-converters/src/test/resources/test3b.conf
deleted file mode 100644
index 53ca4b3..0000000
--- a/streams-components/streams-converters/src/test/resources/test3b.conf
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-id = "actorid"
-displayName = "Jorge"
diff --git a/streams-components/streams-converters/src/test/resources/test4.conf b/streams-components/streams-converters/src/test/resources/test4.conf
deleted file mode 100644
index 8d9e779..0000000
--- a/streams-components/streams-converters/src/test/resources/test4.conf
+++ /dev/null
@@ -1,19 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-profile.id = ${actor.id}
-profile.name = ${actor.name}
-profile.username = ${actor.screenName}
-profile.description = ${actor.summary}
diff --git a/streams-components/streams-filters/README.md b/streams-components/streams-filters/README.md
deleted file mode 100644
index 3d8a70a..0000000
--- a/streams-components/streams-filters/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-filters
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-components/streams-filters/pom.xml b/streams-components/streams-filters/pom.xml
deleted file mode 100644
index 2d7bb7b..0000000
--- a/streams-components/streams-filters/pom.xml
+++ /dev/null
@@ -1,97 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-components</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>streams-filters</artifactId>
-
- <name>streams-filters</name>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-verbs</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes combine.children="append">
- <exclude>src/test/resources/actor.json</exclude>
- <exclude>src/test/resources/follow.json</exclude>
- <exclude>src/test/resources/object.json</exclude>
- <exclude>src/test/resources/post.json</exclude>
- <exclude>src/test/resources/provider.json</exclude>
- <exclude>src/test/resources/targetrequired.json</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-</project>
diff --git a/streams-components/streams-filters/src/main/java/org/apache/streams/filters/VerbDefinitionDropFilter.java b/streams-components/streams-filters/src/main/java/org/apache/streams/filters/VerbDefinitionDropFilter.java
deleted file mode 100644
index 08106c4..0000000
--- a/streams-components/streams-filters/src/main/java/org/apache/streams/filters/VerbDefinitionDropFilter.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.filters;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.verbs.VerbDefinition;
-import org.apache.streams.verbs.VerbDefinitionMatchUtil;
-import org.apache.streams.verbs.VerbDefinitionResolver;
-
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-
-/**
- * Checks one or more verb definitions against a stream of Activity documents, and drops any activities
- * which match the filter criteria.
- */
-public class VerbDefinitionDropFilter implements StreamsProcessor {
-
- public static final String STREAMS_ID = "VerbDefinitionDropFilter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(VerbDefinitionDropFilter.class);
-
- protected Set<VerbDefinition> verbDefinitionSet;
- protected VerbDefinitionResolver resolver;
-
- public VerbDefinitionDropFilter() {
- // get with reflection
- }
-
- public VerbDefinitionDropFilter(Set<VerbDefinition> verbDefinitionSet) {
- this();
- this.verbDefinitionSet = verbDefinitionSet;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = new ArrayList<>();
-
- LOGGER.debug("{} filtering {}", STREAMS_ID, entry.getDocument().getClass());
-
- Activity activity;
-
- Preconditions.checkArgument(entry.getDocument() instanceof Activity);
-
- activity = (Activity) entry.getDocument();
-
- if (!VerbDefinitionMatchUtil.match(activity, this.verbDefinitionSet)) {
- result.add(entry);
- }
-
- return result;
- }
-
- @Override
- public void prepare(Object configuration) {
- if ( verbDefinitionSet != null) {
- resolver = new VerbDefinitionResolver(verbDefinitionSet);
- } else {
- resolver = new VerbDefinitionResolver();
- }
- Objects.requireNonNull(resolver);
- }
-
- @Override
- public void cleanUp() {
- // noOp
- }
-
-}
diff --git a/streams-components/streams-filters/src/main/java/org/apache/streams/filters/VerbDefinitionKeepFilter.java b/streams-components/streams-filters/src/main/java/org/apache/streams/filters/VerbDefinitionKeepFilter.java
deleted file mode 100644
index 89efb31..0000000
--- a/streams-components/streams-filters/src/main/java/org/apache/streams/filters/VerbDefinitionKeepFilter.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.filters;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.verbs.VerbDefinition;
-import org.apache.streams.verbs.VerbDefinitionMatchUtil;
-import org.apache.streams.verbs.VerbDefinitionResolver;
-
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-
-/**
- * Checks one or more verb definitions against a stream of Activity documents, and drops any activities
- * which do not match the filter criteria.
- */
-public class VerbDefinitionKeepFilter implements StreamsProcessor {
-
- public static final String STREAMS_ID = "VerbDefinitionKeepFilter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(VerbDefinitionKeepFilter.class);
-
- protected Set<VerbDefinition> verbDefinitionSet;
- protected VerbDefinitionResolver resolver;
-
- public VerbDefinitionKeepFilter() {
- // get with reflection
- }
-
- public VerbDefinitionKeepFilter(Set<VerbDefinition> verbDefinitionSet) {
- this();
- this.verbDefinitionSet = verbDefinitionSet;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = new ArrayList<>();
-
- LOGGER.debug("{} filtering {}", STREAMS_ID, entry.getDocument().getClass());
-
- Activity activity;
-
- Preconditions.checkArgument(entry.getDocument() instanceof Activity);
-
- activity = (Activity) entry.getDocument();
-
- if (VerbDefinitionMatchUtil.match(activity, this.verbDefinitionSet)) {
- result.add(entry);
- }
-
- return result;
- }
-
- @Override
- public void prepare(Object configuration) {
- if ( verbDefinitionSet != null ) {
- resolver = new VerbDefinitionResolver(verbDefinitionSet);
- } else {
- resolver = new VerbDefinitionResolver();
- }
- Objects.requireNonNull(resolver);
- }
-
- @Override
- public void cleanUp() {
- // noOp
- }
-
-}
diff --git a/streams-components/streams-filters/src/src/markdown/index.md b/streams-components/streams-filters/src/src/markdown/index.md
deleted file mode 100644
index 2ba2765..0000000
--- a/streams-components/streams-filters/src/src/markdown/index.md
+++ /dev/null
@@ -1,8 +0,0 @@
-streams-processor-filters
-=====================
-
-Apply a filter to an activity stream, dropping any document that does not match the criteria
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-components/streams-filters/src/test/java/org/apache/streams/filters/test/VerbDefinitionFilterTest.java b/streams-components/streams-filters/src/test/java/org/apache/streams/filters/test/VerbDefinitionFilterTest.java
deleted file mode 100644
index 48b9204..0000000
--- a/streams-components/streams-filters/src/test/java/org/apache/streams/filters/test/VerbDefinitionFilterTest.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.filters.test;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.filters.VerbDefinitionDropFilter;
-import org.apache.streams.filters.VerbDefinitionKeepFilter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.verbs.VerbDefinition;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Test;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests for {$link: org.apache.streams.verbs.VerbDefinitionResolver}
- */
-public class VerbDefinitionFilterTest {
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- /**
- * Test verb match filter alone
- */
- @Test
- public void testVerbMatchFilter() throws Exception {
- VerbDefinition definition = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/post.json"), VerbDefinition.class);
- VerbDefinitionKeepFilter filter = new VerbDefinitionKeepFilter(Stream.of(definition).collect(Collectors.toSet()));
- filter.prepare(null);
- StreamsDatum datum1 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"notpost\"}\n", Activity.class));
- List<StreamsDatum> result1 = filter.process(datum1);
- assert result1.size() == 0;
- StreamsDatum datum2 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\"}\n", Activity.class));
- List<StreamsDatum> result2 = filter.process(datum2);
- assert result2.size() == 1;
- }
-
- /**
- * Test provider filter, if provider has wrong type it should not pass
- */
- @Test
- public void testProviderFilter() throws Exception {
- VerbDefinition definition = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/provider.json"), VerbDefinition.class);
- VerbDefinitionKeepFilter filter = new VerbDefinitionKeepFilter(Stream.of(definition).collect(Collectors.toSet()));
- filter.prepare(null);
- StreamsDatum datum1 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"provider\":{\"id\":\"providerId\",\"objectType\":\"product\"}}\n", Activity.class));
- List<StreamsDatum> result1 = filter.process(datum1);
- assert result1.size() == 0;
- StreamsDatum datum2 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"provider\":{\"id\":\"providerId\",\"objectType\":\"application\"}}\n\n", Activity.class));
- List<StreamsDatum> result2 = filter.process(datum2);
- assert result2.size() == 1;
- }
-
- /**
- * Test actor filter, if actor isn't present it should not pass
- */
- @Test
- public void testActorFilter() throws Exception {
- VerbDefinition definition = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/actor.json"), VerbDefinition.class);
- VerbDefinitionKeepFilter filter = new VerbDefinitionKeepFilter(Stream.of(definition).collect(Collectors.toSet()));
- filter.prepare(null);
- StreamsDatum datum1 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"actor\":{\"id\":\"actorId\",\"objectType\":\"page\"}}\n", Activity.class));
- List<StreamsDatum> result1 = filter.process(datum1);
- assert result1.size() == 0;
- StreamsDatum datum2 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"actor\":{\"id\":\"actorId\",\"objectType\":\"person\"}}}\n\n", Activity.class));
- List<StreamsDatum> result2 = filter.process(datum2);
- assert result2.size() == 1;
-
- }
-
- /**
- * Test object filter, if object doesn't have a type it should not pass
- */
- @Test
- public void testObjectFilter() throws Exception {
- VerbDefinition definition = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/object.json"), VerbDefinition.class);
- VerbDefinitionKeepFilter filter = new VerbDefinitionKeepFilter(Stream.of(definition).collect(Collectors.toSet()));
- filter.prepare(null);
- StreamsDatum datum1 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"object\":{\"id\":\"objectId\"}}\n", Activity.class));
- List<StreamsDatum> result1 = filter.process(datum1);
- assert result1.size() == 0;
- StreamsDatum datum2 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"object\":{\"id\":\"objectId\",\"objectType\":\"task\"}}}\n\n", Activity.class));
- List<StreamsDatum> result2 = filter.process(datum2);
- assert result2.size() == 1;
- }
-
- /**
- * Test actor and object filter together
- */
- @Test
- public void testMultiFilter() throws Exception {
- VerbDefinition definition = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/follow.json"), VerbDefinition.class);
- VerbDefinitionKeepFilter filter = new VerbDefinitionKeepFilter(Stream.of(definition).collect(Collectors.toSet()));
- filter.prepare(null);
- StreamsDatum datum1 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"follow\",\"actor\":{\"id\":\"actorId\",\"objectType\":\"page\"}}}\n", Activity.class));
- List<StreamsDatum> result1 = filter.process(datum1);
- assert result1.size() == 0;
- StreamsDatum datum2 = new StreamsDatum(mapper.readValue("{\"id\":\"2\",\"verb\":\"follow\",\"object\":{\"id\":\"objectId\",\"objectType\":\"page\"}}}\n", Activity.class));
- List<StreamsDatum> result2 = filter.process(datum2);
- assert result2.size() == 0;
- StreamsDatum datum3 = new StreamsDatum(mapper.readValue("{\"id\":\"3\",\"verb\":\"follow\",\"actor\":{\"id\":\"actorId\",\"objectType\":\"page\"},\"object\":{\"id\":\"objectId\",\"objectType\":\"page\"}}}\n", Activity.class));
- List<StreamsDatum> result3 = filter.process(datum3);
- assert result3.size() == 1;
-
- }
-
- /**
- * Test targetRequired
- */
- @Test
- public void testTargetRequired() throws Exception {
- VerbDefinition definition = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/targetrequired.json"), VerbDefinition.class);
- VerbDefinitionKeepFilter filter = new VerbDefinitionKeepFilter(Stream.of(definition).collect(Collectors.toSet()));
- filter.prepare(null);
- StreamsDatum datum1 = new StreamsDatum(mapper.readValue("{\"id\":\"id\",\"verb\":\"post\",\"object\":{\"id\":\"objectId\",\"objectType\":\"task\"}}\n", Activity.class));
- List<StreamsDatum> result1 = filter.process(datum1);
- assert result1.size() == 0;
- StreamsDatum datum2 = new StreamsDatum(mapper.readValue("{\"id\":\"id\",\"verb\":\"post\",\"target\":{\"id\":\"targetId\",\"objectType\":\"group\"}}\n", Activity.class));
- List<StreamsDatum> result2 = filter.process(datum2);
- assert result2.size() == 1;
- }
-
- /**
- * Test that wildcard verb definition matches every item
- */
- @Test
- public void testAllWildcard() throws Exception {
- VerbDefinition definition = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/post.json"), VerbDefinition.class);
- VerbDefinitionKeepFilter filter = new VerbDefinitionKeepFilter(Stream.of(definition).collect(Collectors.toSet()));
- filter.prepare(null);
- StreamsDatum datum1 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"notpost\"}\n", Activity.class));
- List<StreamsDatum> result1 = filter.process(datum1);
- assert result1.size() == 0;
- StreamsDatum datum2 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\"}\n", Activity.class));
- List<StreamsDatum> result2 = filter.process(datum2);
- assert result2.size() == 1;
- StreamsDatum datum3 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"provider\":{\"id\":\"providerId\",\"objectType\":\"product\"}}\n", Activity.class));
- List<StreamsDatum> result3 = filter.process(datum3);
- assert result3.size() == 1;
- StreamsDatum datum4 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"provider\":{\"id\":\"providerId\",\"objectType\":\"application\"}}\n\n", Activity.class));
- List<StreamsDatum> result4 = filter.process(datum4);
- assert result4.size() == 1;
- StreamsDatum datum5 = new StreamsDatum(mapper.readValue("{\"id\":\"id\",\"verb\":\"post\",\"object\":{\"id\":\"objectId\",\"objectType\":\"task\"}}\n", Activity.class));
- List<StreamsDatum> result5 = filter.process(datum5);
- assert result5.size() == 1;
- StreamsDatum datum6 = new StreamsDatum(mapper.readValue("{\"id\":\"id\",\"verb\":\"post\",\"target\":{\"id\":\"targetId\",\"objectType\":\"group\"}}\n", Activity.class));
- List<StreamsDatum> result6 = filter.process(datum6);
- assert result6.size() == 1;
- StreamsDatum datum7 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"actor\":{\"id\":\"actorId\",\"objectType\":\"page\"}}\n", Activity.class));
- List<StreamsDatum> result7 = filter.process(datum7);
- assert result7.size() == 1;
- }
-
- /**
- * Test that multiple verb definitions chain properly
- */
- @Test
- public void testAllMultipleDefinition() throws Exception {
- VerbDefinition provider = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/provider.json"), VerbDefinition.class);
- VerbDefinition actor = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/actor.json"), VerbDefinition.class);
- VerbDefinition object = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/object.json"), VerbDefinition.class);
- VerbDefinition target = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/targetrequired.json"), VerbDefinition.class);
- VerbDefinition follow = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/follow.json"), VerbDefinition.class);
- VerbDefinitionKeepFilter filter = new VerbDefinitionKeepFilter(Stream.of(provider,actor,object,target,follow).collect(Collectors.toSet()));
- filter.prepare(null);
- StreamsDatum datum1 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"notpost\"}\n", Activity.class));
- List<StreamsDatum> result1 = filter.process(datum1);
- assert result1.size() == 0;
- StreamsDatum datum2 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\"}\n", Activity.class));
- List<StreamsDatum> result2 = filter.process(datum2);
- assert result2.size() == 1;
- StreamsDatum datum3 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"provider\":{\"id\":\"providerId\",\"objectType\":\"product\"}}\n", Activity.class));
- List<StreamsDatum> result3 = filter.process(datum3);
- assert result3.size() == 1;
- StreamsDatum datum4 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"provider\":{\"id\":\"providerId\",\"objectType\":\"application\"}}\n\n", Activity.class));
- List<StreamsDatum> result4 = filter.process(datum4);
- assert result4.size() == 1;
- StreamsDatum datum5 = new StreamsDatum(mapper.readValue("{\"id\":\"id\",\"verb\":\"post\",\"object\":{\"id\":\"objectId\",\"objectType\":\"task\"}}\n", Activity.class));
- List<StreamsDatum> result5 = filter.process(datum5);
- assert result5.size() == 1;
- StreamsDatum datum6 = new StreamsDatum(mapper.readValue("{\"id\":\"id\",\"verb\":\"post\",\"target\":{\"id\":\"targetId\",\"objectType\":\"group\"}}\n", Activity.class));
- List<StreamsDatum> result6 = filter.process(datum6);
- assert result6.size() == 1;
- StreamsDatum datum7 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"actor\":{\"id\":\"actorId\",\"objectType\":\"page\"}}\n", Activity.class));
- List<StreamsDatum> result7 = filter.process(datum7);
- assert result7.size() == 1;
- StreamsDatum datum9 = new StreamsDatum(mapper.readValue("{\"id\":\"3\",\"verb\":\"follow\",\"actor\":{\"id\":\"actorId\",\"objectType\":\"page\"},\"object\":{\"id\":\"objectId\",\"objectType\":\"page\"}}}\n", Activity.class));
- List<StreamsDatum> result9 = filter.process(datum9);
- assert result9.size() == 1;
- }
-
- /**
- * Test verb drop filter alone
- */
- @Test
- public void testVerbDropFilter() throws Exception {
- VerbDefinition definition = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/post.json"), VerbDefinition.class);
- VerbDefinitionDropFilter filter = new VerbDefinitionDropFilter(Stream.of(definition).collect(Collectors.toSet()));
- filter.prepare(null);
- StreamsDatum datum1 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"notpost\"}\n", Activity.class));
- List<StreamsDatum> result1 = filter.process(datum1);
- assert result1.size() == 1;
- StreamsDatum datum2 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\"}\n", Activity.class));
- List<StreamsDatum> result2 = filter.process(datum2);
- assert result2.size() == 0;
- }
-
- /**
- * Test that wildcard verb definition will drop every item
- */
- @Test
- public void testDropAllWildcard() throws Exception {
- VerbDefinition definition = mapper.readValue(VerbDefinitionFilterTest.class.getResourceAsStream("/post.json"), VerbDefinition.class);
- VerbDefinitionDropFilter filter = new VerbDefinitionDropFilter(Stream.of(definition).collect(Collectors.toSet()));
- filter.prepare(null);
- StreamsDatum datum1 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"notpost\"}\n", Activity.class));
- List<StreamsDatum> result1 = filter.process(datum1);
- assert result1.size() == 1;
- StreamsDatum datum2 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\"}\n", Activity.class));
- List<StreamsDatum> result2 = filter.process(datum2);
- assert result2.size() == 0;
- StreamsDatum datum3 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"provider\":{\"id\":\"providerId\",\"objectType\":\"product\"}}\n", Activity.class));
- List<StreamsDatum> result3 = filter.process(datum3);
- assert result3.size() == 0;
- StreamsDatum datum4 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"provider\":{\"id\":\"providerId\",\"objectType\":\"application\"}}\n\n", Activity.class));
- List<StreamsDatum> result4 = filter.process(datum4);
- assert result4.size() == 0;
- StreamsDatum datum5 = new StreamsDatum(mapper.readValue("{\"id\":\"id\",\"verb\":\"post\",\"object\":{\"id\":\"objectId\",\"objectType\":\"task\"}}\n", Activity.class));
- List<StreamsDatum> result5 = filter.process(datum5);
- assert result5.size() == 0;
- StreamsDatum datum6 = new StreamsDatum(mapper.readValue("{\"id\":\"id\",\"verb\":\"post\",\"target\":{\"id\":\"targetId\",\"objectType\":\"group\"}}\n", Activity.class));
- List<StreamsDatum> result6 = filter.process(datum6);
- assert result6.size() == 0;
- StreamsDatum datum7 = new StreamsDatum(mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"actor\":{\"id\":\"actorId\",\"objectType\":\"page\"}}\n", Activity.class));
- List<StreamsDatum> result7 = filter.process(datum7);
- assert result7.size() == 0;
- }
-
-}
\ No newline at end of file
diff --git a/streams-components/streams-filters/src/test/resources/actor.json b/streams-components/streams-filters/src/test/resources/actor.json
deleted file mode 100644
index f7b62a8..0000000
--- a/streams-components/streams-filters/src/test/resources/actor.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/targetrequired",
- "value": "post",
- "objects": [
- {
- "actor": "person"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-components/streams-filters/src/test/resources/follow.json b/streams-components/streams-filters/src/test/resources/follow.json
deleted file mode 100644
index 082c65f..0000000
--- a/streams-components/streams-filters/src/test/resources/follow.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/targetrequired",
- "value": "follow",
- "objects": [
- {
- "actorRequired": "true",
- "actor": "page",
- "object": "page",
- "objectRequired": "true"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-components/streams-filters/src/test/resources/object.json b/streams-components/streams-filters/src/test/resources/object.json
deleted file mode 100644
index 36eed78..0000000
--- a/streams-components/streams-filters/src/test/resources/object.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/targetrequired",
- "value": "post",
- "objects": [
- {
- "object": "task"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-components/streams-filters/src/test/resources/post.json b/streams-components/streams-filters/src/test/resources/post.json
deleted file mode 100644
index 33b22c1..0000000
--- a/streams-components/streams-filters/src/test/resources/post.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/post",
- "value": "post",
- "objects": [
- {
- "actor": "*",
- "object": "*",
- "provider": "*",
- "target": "*"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-components/streams-filters/src/test/resources/provider.json b/streams-components/streams-filters/src/test/resources/provider.json
deleted file mode 100644
index c5ec184..0000000
--- a/streams-components/streams-filters/src/test/resources/provider.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/targetrequired",
- "value": "post",
- "objects": [
- {
- "provider": "application"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-components/streams-filters/src/test/resources/targetrequired.json b/streams-components/streams-filters/src/test/resources/targetrequired.json
deleted file mode 100644
index 2d5924b..0000000
--- a/streams-components/streams-filters/src/test/resources/targetrequired.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/targetrequired",
- "value": "post",
- "objects": [
- {
- "actor": "*",
- "object": "*",
- "target": "*",
- "targetRequired": "true"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-components/streams-http/README.md b/streams-components/streams-http/README.md
deleted file mode 100644
index 5e3ae0b..0000000
--- a/streams-components/streams-http/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-http
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-components/streams-http/pom.xml b/streams-components/streams-http/pom.xml
deleted file mode 100644
index 221184e..0000000
--- a/streams-components/streams-http/pom.xml
+++ /dev/null
@@ -1,159 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-components</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>streams-http</artifactId>
-
- <name>streams-http</name>
-
- <description>HTTP Utility Components</description>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.http</targetPackage>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-converters</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/persist/SimpleHTTPPostPersistWriter.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/persist/SimpleHTTPPostPersistWriter.java
deleted file mode 100644
index f4c1a5c..0000000
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/persist/SimpleHTTPPostPersistWriter.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.components.http.persist;
-
-import org.apache.streams.components.http.HttpPersistWriterConfiguration;
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-public class SimpleHTTPPostPersistWriter implements StreamsPersistWriter {
-
- private static final String STREAMS_ID = "SimpleHTTPPostPersistWriter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SimpleHTTPPostPersistWriter.class);
-
- protected ObjectMapper mapper;
-
- protected URIBuilder uriBuilder;
-
- protected CloseableHttpClient httpclient;
-
- protected HttpPersistWriterConfiguration configuration;
-
- protected String authHeader;
-
- public SimpleHTTPPostPersistWriter() {
- this(new ComponentConfigurator<>(HttpPersistWriterConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("http")));
- }
-
- public SimpleHTTPPostPersistWriter(HttpPersistWriterConfiguration configuration) {
- this.configuration = configuration;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void write(StreamsDatum entry) {
-
- ObjectNode payload;
- try {
- payload = preparePayload(entry);
- } catch ( Exception ex ) {
- LOGGER.warn("Exception preparing payload, using empty payload");
- payload = mapper.createObjectNode();
- }
-
-
- Map<String, String> params = prepareParams(entry);
-
- URI uri = prepareURI(params);
-
- HttpPost httppost = prepareHttpPost(uri, payload);
-
- ObjectNode result = executePost(httppost);
-
- try {
- LOGGER.debug(mapper.writeValueAsString(result));
- } catch (JsonProcessingException ex) {
- LOGGER.warn("Non-json response", ex.getMessage());
- }
- }
-
- /**
- Override this to alter request URI.
- */
- protected URI prepareURI(Map<String, String> params) {
- URI uri = null;
- for (Map.Entry<String,String> param : params.entrySet()) {
- uriBuilder = uriBuilder.setParameter(param.getKey(), param.getValue());
- }
- try {
- uri = uriBuilder.build();
- } catch (URISyntaxException ex) {
- LOGGER.error("URI error {}", uriBuilder.toString());
- }
- return uri;
- }
-
- /**
- Override this to add parameters to the request.
- */
- protected Map<String, String> prepareParams(StreamsDatum entry) {
- return new HashMap<>();
- }
-
- /**
- Override this to alter json payload on to the request.
- */
- protected ObjectNode preparePayload(StreamsDatum entry) throws Exception {
-
- if ( entry.getDocument() != null ) {
- if ( entry.getDocument() instanceof ObjectNode ) {
- return (ObjectNode) entry.getDocument();
- } else {
- return mapper.convertValue(entry.getDocument(), ObjectNode.class);
- }
- } else {
- return null;
- }
- }
-
- /**
- Override this to add headers to the request.
- */
- public HttpPost prepareHttpPost(URI uri, ObjectNode payload) {
- HttpPost httppost = new HttpPost(uri);
- httppost.addHeader("content-type", this.configuration.getContentType());
- httppost.addHeader("accept-charset", "UTF-8");
- if (StringUtils.isNotBlank(authHeader)) {
- httppost.addHeader("Authorization", "Basic " + authHeader);
- }
- try {
- String entity = mapper.writeValueAsString(payload);
- httppost.setEntity(new StringEntity(entity));
- } catch (JsonProcessingException | UnsupportedEncodingException ex) {
- LOGGER.warn(ex.getMessage());
- }
- return httppost;
- }
-
- protected ObjectNode executePost(HttpPost httpPost) {
-
- Objects.requireNonNull(httpPost);
-
- ObjectNode result = null;
-
- CloseableHttpResponse response = null;
-
- String entityString;
- try {
- response = httpclient.execute(httpPost);
- HttpEntity entity = response.getEntity();
- // TODO: handle retry
- if (response.getStatusLine() != null && response.getStatusLine().getStatusCode() >= HttpStatus.SC_OK && entity != null) {
- entityString = EntityUtils.toString(entity);
- result = mapper.readValue(entityString, ObjectNode.class);
- }
- } catch (IOException ex) {
- LOGGER.error("IO error:\n{}\n{}\n{}", httpPost.toString(), response, ex.getMessage());
- } finally {
- try {
- if (response != null) {
- response.close();
- }
- } catch (IOException ignored) {
- LOGGER.trace("IOException", ignored);
- }
- }
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- mapper = StreamsJacksonMapper.getInstance();
-
- uriBuilder = new URIBuilder()
- .setScheme(this.configuration.getProtocol())
- .setHost(this.configuration.getHostname())
- .setPort(this.configuration.getPort().intValue())
- .setPath(this.configuration.getResourcePath());
-
- if (StringUtils.isNotBlank(configuration.getAccessToken()) ) {
- uriBuilder = uriBuilder.addParameter("access_token", configuration.getAccessToken());
- }
- if (StringUtils.isNotBlank(configuration.getUsername()) && StringUtils.isNotBlank(configuration.getPassword())) {
- String string = configuration.getUsername() + ":" + configuration.getPassword();
- authHeader = Base64.encodeBase64String(string.getBytes());
- }
-
- httpclient = HttpClients.createDefault();
-
- }
-
- @Override
- public void cleanUp() {
-
- LOGGER.info("shutting down SimpleHTTPPostPersistWriter");
- try {
- httpclient.close();
- } catch (IOException ex) {
- LOGGER.error(ex.getMessage());
- } finally {
- try {
- httpclient.close();
- } catch (IOException e2) {
- LOGGER.error(e2.getMessage());
- } finally {
- httpclient = null;
- }
- }
- }
-}
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
deleted file mode 100644
index ddc1ec5..0000000
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPGetProcessor.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.components.http.processor;
-
-import org.apache.streams.components.http.HttpProcessorConfiguration;
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Processor retrieves contents from an known url and stores the resulting object in an extension field.
- */
-public class SimpleHTTPGetProcessor implements StreamsProcessor {
-
- private static final String STREAMS_ID = "SimpleHTTPGetProcessor";
-
- // from root config id
- private static final String EXTENSION = "account_type";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SimpleHTTPGetProcessor.class);
-
- protected ObjectMapper mapper;
-
- protected URIBuilder uriBuilder;
-
- protected CloseableHttpClient httpclient;
-
- protected HttpProcessorConfiguration configuration;
-
- protected String authHeader;
-
- /**
- * SimpleHTTPGetProcessor constructor - resolves HttpProcessorConfiguration from JVM 'http'.
- */
- public SimpleHTTPGetProcessor() {
- this(new ComponentConfigurator<>(HttpProcessorConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("http")));
- }
-
- /**
- * SimpleHTTPGetProcessor constructor - uses provided HttpProcessorConfiguration.
- */
- public SimpleHTTPGetProcessor(HttpProcessorConfiguration processorConfiguration) {
- LOGGER.info("creating SimpleHTTPGetProcessor");
- LOGGER.info(processorConfiguration.toString());
- this.configuration = processorConfiguration;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- /**
- Override this to store a result other than exact json representation of response.
- */
- protected ObjectNode prepareExtensionFragment(String entityString) {
-
- try {
- return mapper.readValue(entityString, ObjectNode.class);
- } catch (IOException ex) {
- LOGGER.warn(ex.getMessage());
- return null;
- }
- }
-
- /**
- Override this to place result in non-standard location on document.
- */
- protected ObjectNode getRootDocument(StreamsDatum datum) {
-
- try {
- String json = datum.getDocument() instanceof String
- ? (String) datum.getDocument()
- : mapper.writeValueAsString(datum.getDocument());
- return mapper.readValue(json, ObjectNode.class);
- } catch (IOException ex) {
- LOGGER.warn(ex.getMessage());
- return null;
- }
-
- }
-
- /**
- Override this to place result in non-standard location on document.
- */
- protected ActivityObject getEntityToExtend(ObjectNode rootDocument) {
-
- if ( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY)) {
- return mapper.convertValue(rootDocument, ActivityObject.class);
- } else {
- return mapper.convertValue(rootDocument.get(this.configuration.getEntity().toString()), ActivityObject.class);
- }
- }
-
- /**
- Override this to place result in non-standard location on document.
- */
- protected ObjectNode setEntityToExtend(ObjectNode rootDocument, ActivityObject activityObject) {
-
- if ( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY)) {
- return mapper.convertValue(activityObject, ObjectNode.class);
- } else {
- rootDocument.set(this.configuration.getEntity().toString(), mapper.convertValue(activityObject, ObjectNode.class));
- }
-
- return rootDocument;
-
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = new ArrayList<>();
-
- ObjectNode rootDocument = getRootDocument(entry);
-
- Map<String, String> params = prepareParams(entry);
-
- URI uri = prepareURI(params);
-
- HttpGet httpget = prepareHttpGet(uri);
-
- CloseableHttpResponse response = null;
-
- String entityString = null;
- try {
- response = httpclient.execute(httpget);
- HttpEntity entity = response.getEntity();
- // TODO: handle retry
- if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
- entityString = EntityUtils.toString(entity);
- }
- } catch (IOException ex) {
- LOGGER.error("IO error:\n{}\n{}\n{}", uri.toString(), response, ex.getMessage());
- return result;
- } finally {
- try {
- if (response != null) {
- response.close();
- }
- } catch (IOException ignored) {
- LOGGER.trace("IOException", ignored);
- }
- }
-
- if( entityString == null ) {
- return result;
- }
-
- LOGGER.debug(entityString);
-
- ObjectNode extensionFragment = prepareExtensionFragment(entityString);
-
- ActivityObject extensionEntity = getEntityToExtend(rootDocument);
-
- ExtensionUtil.getInstance().addExtension(extensionEntity, this.configuration.getExtension(), extensionFragment);
-
- rootDocument = setEntityToExtend(rootDocument, extensionEntity);
-
- entry.setDocument(rootDocument);
-
- result.add(entry);
-
- return result;
-
- }
-
- /**
- Override this to alter request URI.
- */
- protected URI prepareURI(Map<String, String> params) {
-
- URI uri = null;
- for ( Map.Entry<String,String> param : params.entrySet()) {
- uriBuilder = uriBuilder.setParameter(param.getKey(), param.getValue());
- }
- try {
- uri = uriBuilder.build();
- } catch (URISyntaxException ex) {
- LOGGER.error("URI error {}", uriBuilder.toString());
- }
- return uri;
- }
-
- /**
- Override this to add parameters to the request.
- */
- protected Map<String, String> prepareParams(StreamsDatum entry) {
- return new HashMap<>();
- }
-
- /**
- Override this to set a payload on the request.
- */
- protected ObjectNode preparePayload(StreamsDatum entry) {
- return null;
- }
-
- /**
- * Override this to set the URI for the request or modify headers.
- * @param uri uri
- * @return result
- */
- public HttpGet prepareHttpGet(URI uri) {
- HttpGet httpget = new HttpGet(uri);
- httpget.addHeader("content-type", this.configuration.getContentType());
- if (StringUtils.isNotBlank(authHeader)) {
- httpget.addHeader("Authorization", String.format("Basic %s", authHeader));
- }
- return httpget;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- mapper = StreamsJacksonMapper.getInstance();
-
- uriBuilder = new URIBuilder()
- .setScheme(this.configuration.getProtocol())
- .setHost(this.configuration.getHostname())
- .setPath(this.configuration.getResourcePath());
-
- if (StringUtils.isNotBlank(configuration.getAccessToken()) ) {
- uriBuilder = uriBuilder.addParameter("access_token", configuration.getAccessToken());
- }
- if (StringUtils.isNotBlank(configuration.getUsername())
- &&
- StringUtils.isNotBlank(configuration.getPassword())) {
- String string = configuration.getUsername() + ":" + configuration.getPassword();
- authHeader = Base64.encodeBase64String(string.getBytes());
- }
- httpclient = HttpClients.createDefault();
- }
-
- @Override
- public void cleanUp() {
- LOGGER.info("shutting down SimpleHTTPGetProcessor");
- try {
- httpclient.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- } finally {
- try {
- httpclient.close();
- } catch (IOException e2) {
- e2.printStackTrace();
- } finally {
- httpclient = null;
- }
- }
- }
-}
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPPostProcessor.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPPostProcessor.java
deleted file mode 100644
index 6901b4f..0000000
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/processor/SimpleHTTPPostProcessor.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.components.http.processor;
-
-import org.apache.streams.components.http.HttpProcessorConfiguration;
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Processor retrieves contents from an known url and stores the resulting object in an extension field.
- */
-public class SimpleHTTPPostProcessor implements StreamsProcessor {
-
- private static final String STREAMS_ID = "SimpleHTTPPostProcessor";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SimpleHTTPPostProcessor.class);
-
- protected ObjectMapper mapper;
-
- protected URIBuilder uriBuilder;
-
- protected CloseableHttpClient httpclient;
-
- protected HttpProcessorConfiguration configuration;
-
- protected String authHeader;
-
- /**
- * SimpleHTTPPostProcessor constructor - resolves HttpProcessorConfiguration from JVM 'http'.
- */
- public SimpleHTTPPostProcessor() {
- this(new ComponentConfigurator<>(HttpProcessorConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("http")));
- }
-
- /**
- * SimpleHTTPPostProcessor constructor - uses provided HttpProcessorConfiguration.
- */
- public SimpleHTTPPostProcessor(HttpProcessorConfiguration processorConfiguration) {
- LOGGER.info("creating SimpleHTTPPostProcessor");
- LOGGER.info(processorConfiguration.toString());
- this.configuration = processorConfiguration;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- /**
- Override this to store a result other than exact json representation of response.
- */
- protected ObjectNode prepareExtensionFragment(String entityString) {
-
- try {
- return mapper.readValue(entityString, ObjectNode.class);
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- return null;
- }
- }
-
- /**
- Override this to place result in non-standard location on document.
- */
- protected ObjectNode getRootDocument(StreamsDatum datum) {
-
- try {
- String json = datum.getDocument() instanceof String
- ? (String) datum.getDocument()
- : mapper.writeValueAsString(datum.getDocument());
- return mapper.readValue(json, ObjectNode.class);
- } catch (JsonProcessingException ex) {
- LOGGER.warn("JsonProcessingException", ex);
- return null;
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- return null;
- }
-
- }
-
- /**
- Override this to place result in non-standard location on document.
- */
- protected ActivityObject getEntityToExtend(ObjectNode rootDocument) {
-
- if ( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY)) {
- return mapper.convertValue(rootDocument, ActivityObject.class);
- } else {
- return mapper.convertValue(rootDocument.get(this.configuration.getEntity().toString()), ActivityObject.class);
- }
- }
-
- /**
- Override this to place result in non-standard location on document.
- */
- protected ObjectNode setEntityToExtend(ObjectNode rootDocument, ActivityObject activityObject) {
-
- if ( this.configuration.getEntity().equals(HttpProcessorConfiguration.Entity.ACTIVITY)) {
- return mapper.convertValue(activityObject, ObjectNode.class);
- } else {
- rootDocument.set(this.configuration.getEntity().toString(), mapper.convertValue(activityObject, ObjectNode.class));
- }
- return rootDocument;
-
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = new ArrayList<>();
-
- ObjectNode rootDocument = getRootDocument(entry);
-
- Map<String, String> params = prepareParams(entry);
-
- URI uri;
- for ( Map.Entry<String,String> param : params.entrySet() ) {
- uriBuilder = uriBuilder.setParameter(param.getKey(), param.getValue());
- }
- try {
- uri = uriBuilder.build();
- } catch (URISyntaxException ex) {
- LOGGER.error("URI error {}", uriBuilder.toString(), ex);
- return result;
- }
-
- HttpEntity payload = preparePayload(entry);
-
- HttpPost httpPost = prepareHttpPost(uri, payload);
-
- CloseableHttpResponse response = null;
-
- String entityString = null;
- try {
- response = httpclient.execute(httpPost);
- HttpEntity entity = response.getEntity();
- // TODO: handle retry
- if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
- entityString = EntityUtils.toString(entity);
- }
- } catch (IOException ex) {
- LOGGER.error("IO error:\n{}\n{}\n{}", uri.toString(), response, ex);
- return result;
- } finally {
- try {
- if (response != null) {
- response.close();
- }
- } catch (IOException ignored) {
- LOGGER.trace("IOException", ignored);
- }
- }
-
- if ( entityString == null ) {
- return result;
- }
-
- LOGGER.debug(entityString);
-
- ObjectNode extensionFragment = prepareExtensionFragment(entityString);
-
- ActivityObject extensionEntity = getEntityToExtend(rootDocument);
-
- ExtensionUtil.getInstance().addExtension(extensionEntity, this.configuration.getExtension(), extensionFragment);
-
- rootDocument = setEntityToExtend(rootDocument, extensionEntity);
-
- entry.setDocument(rootDocument);
-
- result.add(entry);
-
- return result;
-
- }
-
- /**
- Override this to add parameters to the request.
- */
- protected Map<String, String> prepareParams(StreamsDatum entry) {
- return new HashMap<>();
- }
-
- /**
- Override this to add parameters to the request.
- */
- protected HttpEntity preparePayload(StreamsDatum entry) {
- return new StringEntity("{}",
- ContentType.create("application/json"));
- }
-
- /**
- * Override this to set the URI / entity for the request or modify headers.
- * @param uri uri
- * @param entity entity
- * @return result
- */
- public HttpPost prepareHttpPost(URI uri, HttpEntity entity) {
- HttpPost httpPost = new HttpPost(uri);
- httpPost.addHeader("content-type", this.configuration.getContentType());
- if (StringUtils.isNotBlank(authHeader)) {
- httpPost.addHeader("Authorization", String.format("Basic %s", authHeader));
- }
- httpPost.setEntity(entity);
- return httpPost;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- mapper = StreamsJacksonMapper.getInstance();
-
- uriBuilder = new URIBuilder()
- .setScheme(this.configuration.getProtocol())
- .setHost(this.configuration.getHostname())
- .setPath(this.configuration.getResourcePath());
-
- if (StringUtils.isNotBlank(configuration.getAccessToken()) ) {
- uriBuilder = uriBuilder.addParameter("access_token", configuration.getAccessToken());
- }
- if (StringUtils.isNotBlank(configuration.getUsername())
- && StringUtils.isNotBlank(configuration.getPassword())) {
- String string = configuration.getUsername() + ":" + configuration.getPassword();
- authHeader = Base64.encodeBase64String(string.getBytes());
- }
- httpclient = HttpClients.createDefault();
- }
-
- @Override
- public void cleanUp() {
- LOGGER.info("shutting down SimpleHTTPPostProcessor");
- try {
- httpclient.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- } finally {
- try {
- httpclient.close();
- } catch (IOException e2) {
- LOGGER.error("IOException", e2);
- } finally {
- httpclient = null;
- }
- }
- }
-}
diff --git a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHttpProvider.java b/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHttpProvider.java
deleted file mode 100644
index 9fa0aeb..0000000
--- a/streams-components/streams-http/src/main/java/org/apache/streams/components/http/provider/SimpleHttpProvider.java
+++ /dev/null
@@ -1,352 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.components.http.provider;
-
-import org.apache.streams.components.http.HttpProviderConfiguration;
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.ssl.SSLContextBuilder;
-import org.apache.http.util.EntityUtils;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.math.BigInteger;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.security.KeyManagementException;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * Provider retrieves contents from an known set of urls and passes all resulting objects downstream.
- */
-public class SimpleHttpProvider implements StreamsProvider {
-
- private static final String STREAMS_ID = "SimpleHttpProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SimpleHttpProvider.class);
-
- protected ObjectMapper mapper;
-
- protected URIBuilder uriBuilder;
-
- protected CloseableHttpClient httpclient;
-
- protected HttpProviderConfiguration configuration;
-
- protected volatile Queue<StreamsDatum> providerQueue = new ConcurrentLinkedQueue<>();
-
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- private ExecutorService executor;
-
- /**
- * SimpleHttpProvider constructor - resolves HttpProcessorConfiguration from JVM 'http'.
- */
- public SimpleHttpProvider() {
- this(new ComponentConfigurator<>(HttpProviderConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("http")));
- }
-
- /**
- * SimpleHttpProvider constructor - uses provided HttpProviderConfiguration.
- */
- public SimpleHttpProvider(HttpProviderConfiguration providerConfiguration) {
- LOGGER.info("creating SimpleHttpProvider");
- LOGGER.info(providerConfiguration.toString());
- this.configuration = providerConfiguration;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- /**
- Override this to add parameters to the request.
- */
- protected Map<String, String> prepareParams(StreamsDatum entry) {
- return new HashMap<>();
- }
-
- /**
- * prepareHttpRequest
- * @param uri uri
- * @return result
- */
- public HttpRequestBase prepareHttpRequest(URI uri) {
- HttpRequestBase request;
- if ( configuration.getRequestMethod().equals(HttpProviderConfiguration.RequestMethod.GET)) {
- request = new HttpGet(uri);
- } else if ( configuration.getRequestMethod().equals(HttpProviderConfiguration.RequestMethod.POST)) {
- request = new HttpPost(uri);
- } else {
- // this shouldn't happen because of the default
- request = new HttpGet(uri);
- }
-
- request.addHeader("content-type", this.configuration.getContentType());
-
- return request;
-
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- mapper = StreamsJacksonMapper.getInstance();
-
- uriBuilder = new URIBuilder()
- .setScheme(this.configuration.getProtocol())
- .setHost(this.configuration.getHostname())
- .setPort(this.configuration.getPort().intValue())
- .setPath(this.configuration.getResourcePath());
-
- SSLContextBuilder builder = new SSLContextBuilder();
- SSLConnectionSocketFactory sslsf = null;
- try {
- builder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
- sslsf = new SSLConnectionSocketFactory(
- builder.build(), SSLConnectionSocketFactory.getDefaultHostnameVerifier());
- } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException ex) {
- LOGGER.warn(ex.getMessage());
- }
-
- httpclient = HttpClients.custom().setSSLSocketFactory(
- sslsf).build();
-
- executor = Executors.newSingleThreadExecutor();
-
- }
-
- @Override
- public void cleanUp() {
-
- LOGGER.info("shutting down SimpleHttpProvider");
- this.shutdownAndAwaitTermination(executor);
- try {
- httpclient.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- } finally {
- try {
- httpclient.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- } finally {
- httpclient = null;
- }
- }
- }
-
- @Override
- public void startStream() {
-
- executor.execute(() -> {
-
- readCurrent();
-
- Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
-
- });
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- StreamsResultSet current;
-
- uriBuilder = uriBuilder.setPath(
- String.join("/", uriBuilder.getPath(), configuration.getResource(), configuration.getResourcePostfix())
- );
-
- URI uri;
- try {
- uri = uriBuilder.build();
- } catch (URISyntaxException ex) {
- uri = null;
- }
-
- List<ObjectNode> results = execute(uri);
-
- lock.writeLock().lock();
-
- for ( ObjectNode item : results ) {
- providerQueue.add(newDatum(item));
- }
-
- LOGGER.debug("Creating new result set for {} items", providerQueue.size());
- current = new StreamsResultSet(providerQueue);
-
- return current;
- }
-
- private List<ObjectNode> execute(URI uri) {
-
- Objects.requireNonNull(uri);
-
- List<ObjectNode> results = new ArrayList<>();
-
- HttpRequestBase httpRequest = prepareHttpRequest(uri);
-
- CloseableHttpResponse response = null;
-
- String entityString;
- try {
- response = httpclient.execute(httpRequest);
- HttpEntity entity = response.getEntity();
- // TODO: handle retry
- if (response.getStatusLine().getStatusCode() == 200 && entity != null) {
- entityString = EntityUtils.toString(entity);
- if ( !entityString.equals("{}") && !entityString.equals("[]") ) {
- JsonNode jsonNode = mapper.readValue(entityString, JsonNode.class);
- results = parse(jsonNode);
- }
- }
- } catch (IOException ex) {
- LOGGER.error("IO error:\n{}\n{}\n{}", uri.toString(), response, ex.getMessage());
- } finally {
- try {
- if (response != null) {
- response.close();
- }
- } catch (IOException ignored) {
- LOGGER.trace("IOException", ignored);
- }
- }
- return results;
- }
-
- /**
- Override this to change how entity gets converted to objects.
- */
- protected List<ObjectNode> parse(JsonNode jsonNode) {
-
- List<ObjectNode> results = new ArrayList<>();
-
- if (jsonNode != null && jsonNode instanceof ObjectNode ) {
- results.add((ObjectNode) jsonNode);
- } else if (jsonNode != null && jsonNode instanceof ArrayNode) {
- ArrayNode arrayNode = (ArrayNode) jsonNode;
- Iterator<JsonNode> iterator = arrayNode.elements();
- while (iterator.hasNext()) {
- ObjectNode element = (ObjectNode) iterator.next();
-
- results.add(element);
- }
- }
-
- return results;
- }
-
- /**
- Override this to change how metadata is derived from object.
- */
- private StreamsDatum newDatum(ObjectNode item) {
- try {
- String id = null;
- if ( item.get("id") != null ) {
- id = item.get("id").asText();
- }
- DateTime timestamp = null;
- if ( item.get("timestamp") != null ) {
- timestamp = new DateTime(item.get("timestamp").asText());
- }
- if ( id != null && timestamp != null ) {
- return new StreamsDatum(item, id, timestamp);
- } else if ( id != null ) {
- return new StreamsDatum(item, id);
- } else if ( timestamp != null ) {
- return new StreamsDatum(item, null, timestamp);
- } else {
- return new StreamsDatum(item);
- }
- } catch ( Exception ex ) {
- return new StreamsDatum(item);
- }
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return true;
- }
-
- protected void shutdownAndAwaitTermination(ExecutorService pool) {
- pool.shutdown(); // Disable new tasks from being submitted
- try {
- // Wait a while for existing tasks to terminate
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- pool.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- LOGGER.error("Pool did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- pool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-}
diff --git a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpConfiguration.json b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpConfiguration.json
deleted file mode 100644
index 89712f8..0000000
--- a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpConfiguration.json
+++ /dev/null
@@ -1,61 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "id": "#",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "javaType" : "org.apache.streams.components.http.HttpConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "protocol": {
- "type": "string",
- "description": "Protocol",
- "default": "http"
- },
- "hostname": {
- "type": "string",
- "description": "Hostname",
- "required" : true
- },
- "port": {
- "type": "integer",
- "description": "Port",
- "default": 80
- },
- "resourcePath": {
- "type": "string",
- "description": "Resource Path",
- "required" : true
- },
- "requestMethod": {
- "type": "string",
- "enum": [
- "GET",
- "POST"
- ],
- "default": "GET"
- },
- "content-type": {
- "type": "string",
- "description": "Resource content-type",
- "required" : true,
- "default": "application/json"
- },
- "access_token": {
- "type": "string",
- "description": "Known Access Token",
- "required" : false
- },
- "username": {
- "type": "string",
- "description": "Basic Auth Username",
- "required" : false
- },
- "password": {
- "type": "string",
- "description": "Basic Auth Password",
- "required" : false
- }
- }
-}
\ No newline at end of file
diff --git a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpPersistWriterConfiguration.json b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpPersistWriterConfiguration.json
deleted file mode 100644
index cc421d6..0000000
--- a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpPersistWriterConfiguration.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "id": "#",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "javaType" : "org.apache.streams.components.http.HttpPersistWriterConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": { "$ref": "HttpConfiguration.json" },
- "properties": {
- "resourcePostfixPattern": {
- "type": "string",
- "description": "Pattern for supplying final path for write",
- "required" : false
- }
- }
-}
\ No newline at end of file
diff --git a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProcessorConfiguration.json b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProcessorConfiguration.json
deleted file mode 100644
index 9706761..0000000
--- a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProcessorConfiguration.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.components.http.HttpProcessorConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": { "$ref": "HttpConfiguration.json" },
- "properties": {
- "entity": {
- "type": "string",
- "description": "Entity to extend",
- "enum": [ "activity", "actor", "object", "target" ],
- "required" : true,
- "default": "activity"
- },
- "extension": {
- "type": "string",
- "description": "Extension identifier",
- "required" : true
- },
- "urlField": {
- "type": "string",
- "description": "Field where url is located",
- "required" : true,
- "default": "url"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProviderConfiguration.json b/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProviderConfiguration.json
deleted file mode 100644
index 41c158e..0000000
--- a/streams-components/streams-http/src/main/jsonschema/org/apache/streams/components/http/HttpProviderConfiguration.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.components.http.HttpProviderConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": { "$ref": "HttpConfiguration.json" },
- "properties": {
- "resource": {
- "type": "string",
- "required" : false
- },
- "resourcePostfix": {
- "type": "string",
- "required" : false
- }
- }
-}
\ No newline at end of file
diff --git a/streams-components/streams-http/src/site/markdown/index.md b/streams-components/streams-http/src/site/markdown/index.md
deleted file mode 100644
index e1f203e..0000000
--- a/streams-components/streams-http/src/site/markdown/index.md
+++ /dev/null
@@ -1,26 +0,0 @@
-streams-processor-http
-=====================
-
-Hit an http endpoint and place the result in extensions
-
-Example SimpleHTTPGetProcessor configuration:
-
- "http": {
- "protocol": "http",
- "hostname": "urls.api.twitter.com",
- "port": 9300,
- "resourceUri": "1/urls/count.json"
- }
-
-Example SimpleHTTPPostPersistWriter configuration:
-
- "http": {
- "protocol": "http",
- "hostname": "localhost",
- "port": 7474,
- "resourceUri": "db/data/cypher"
- }
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-components/streams-http/src/test/java/SimpleHTTPPostPersistWriterTest.java b/streams-components/streams-http/src/test/java/SimpleHTTPPostPersistWriterTest.java
deleted file mode 100644
index a74e6bc..0000000
--- a/streams-components/streams-http/src/test/java/SimpleHTTPPostPersistWriterTest.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import org.apache.streams.components.http.HttpPersistWriterConfiguration;
-import org.apache.streams.components.http.persist.SimpleHTTPPostPersistWriter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mockito;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-
-import static org.mockito.Matchers.any;
-
-/**
- * Test for
- * @see org.apache.streams.components.http.persist.SimpleHTTPPostPersistWriter
- */
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({HttpClients.class, CloseableHttpResponse.class, CloseableHttpResponse.class})
-public class SimpleHTTPPostPersistWriterTest {
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- /**
- * test port.
- */
- private static final int PORT = 18080;
-
- /**
- * test hosts.
- */
- private static final String HOSTNAME = "localhost";
-
- /**
- * test protocol.
- */
- private static final String PROTOCOL = "http";
-
- /**
- * CloseableHttpClient mock.
- */
- private CloseableHttpClient client;
-
- /**
- * CloseableHttpClient mock.
- */
- private CloseableHttpResponse response = Mockito.mock(CloseableHttpResponse.class);
-
- /**
- * Our output.
- */
- private ByteArrayOutputStream output;
-
- /**
- * Our input.
- */
- private ByteArrayInputStream input;
-
- @Before
- public void setUp() throws Exception
- {
- /*
- HttpClients mock.
- */
- this.client = PowerMockito.mock(CloseableHttpClient.class);
-
- PowerMockito.mockStatic(HttpClients.class);
-
- PowerMockito.when(HttpClients.createDefault())
- .thenReturn(client);
-
- PowerMockito.when(client.execute(any(HttpUriRequest.class)))
- .thenReturn(response);
-
- Mockito.when(response.getEntity()).thenReturn(null);
- Mockito.doNothing().when(response).close();
-
- }
-
- /**
- * testPersist.
- * @throws Exception
- */
- @Test
- public void testPersist() throws Exception
- {
- HttpPersistWriterConfiguration configuration = new HttpPersistWriterConfiguration();
- configuration.setProtocol(PROTOCOL);
- configuration.setHostname(HOSTNAME);
- configuration.setPort((long) PORT);
- configuration.setResourcePath("/");
-
- /*
- Instance under tests.
- */
- SimpleHTTPPostPersistWriter writer = new SimpleHTTPPostPersistWriter(configuration);
-
- writer.prepare(null);
-
- StreamsDatum testDatum = new StreamsDatum(mapper.readValue("{\"message\":\"ping\"}", ObjectNode.class));
-
- writer.write(testDatum);
-
- Mockito.verify(this.client).execute(any(HttpUriRequest.class));
-
- Mockito.verify(this.response).close();
-
- }
-}
diff --git a/streams-config/README.md b/streams-config/README.md
deleted file mode 100644
index 5e3ae0b..0000000
--- a/streams-config/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-http
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-config/pom.xml b/streams-config/pom.xml
deleted file mode 100644
index 9382489..0000000
--- a/streams-config/pom.xml
+++ /dev/null
@@ -1,153 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>apache-streams</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-config</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Stream and Stream Component General Configuration Objects and Utilities</description>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.juneau</groupId>
- <artifactId>juneau-marshall</artifactId>
- </dependency>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-yaml</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-validator</groupId>
- <artifactId>commons-validator</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>javax.validation</groupId>
- <artifactId>validation-api</artifactId>
- <version>1.1.0.Final</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.config</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>${basedir}/target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-config/src/main/java/org/apache/streams/config/ComponentConfigurator.java b/streams-config/src/main/java/org/apache/streams/config/ComponentConfigurator.java
deleted file mode 100644
index ce88a23..0000000
--- a/streams-config/src/main/java/org/apache/streams/config/ComponentConfigurator.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.config;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigRenderOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-
-/**
- * ComponentConfigurator supplies serializable configuration beans derived from a specified typesafe path or object.
- *
- * <p/>
- * Typically a component will select a 'default' typesafe path to be used if no other path or object is provided.
- *
- * <p/>
- * For example, streams-persist-elasticsearch will use 'elasticsearch' by default, but an implementation
- * such as github.com/apache/streams-examples/local/elasticsearch-reindex
- * can resolve a reader from elasticsearch.source
- * and a writer from elasticsearch.destination.
- *
- */
-public class ComponentConfigurator<T extends Serializable> {
-
- private Class<T> configClass;
-
- public ComponentConfigurator(Class<T> configClass) {
- this.configClass = configClass;
- }
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ComponentConfigurator.class);
-
- private static final ObjectMapper mapper = new ObjectMapper();
-
- /**
- * resolve a serializable configuration pojo from a given typesafe config object.
- * @param typesafeConfig typesafeConfig
- * @return result
- */
- public T detectConfiguration(Config typesafeConfig) {
-
- T pojoConfig = null;
-
- try {
- pojoConfig = mapper.readValue(typesafeConfig.root().render(ConfigRenderOptions.concise()), configClass);
- } catch (Exception ex) {
- ex.printStackTrace();
- LOGGER.warn("Could not parse:", typesafeConfig);
- }
-
- return pojoConfig;
- }
-
- /**
- * resolve a serializable configuration pojo from a portion of the JVM config object.
- * @param subConfig subConfig
- * @return result
- */
- public T detectConfiguration(String subConfig) {
- Config streamsConfig = StreamsConfigurator.getConfig();
- return detectConfiguration( streamsConfig.getConfig(subConfig));
- }
-
- /**
- * resolve a serializable configuration pojo from a portion of a given typesafe config object.
- * @param typesafeConfig typesafeConfig
- * @param subConfig subConfig
- * @return result
- */
- public T detectConfiguration(Config typesafeConfig, String subConfig) {
- return detectConfiguration( typesafeConfig.getConfig(subConfig));
- }
-}
diff --git a/streams-config/src/main/java/org/apache/streams/config/StreamsConfigurator.java b/streams-config/src/main/java/org/apache/streams/config/StreamsConfigurator.java
deleted file mode 100644
index 4c1c14b..0000000
--- a/streams-config/src/main/java/org/apache/streams/config/StreamsConfigurator.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.config;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigRenderOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.MalformedURLException;
-import java.net.URL;
-
-/**
- * StreamsConfigurator supplies the entire typesafe tree to runtimes and modules.
- *
- * StreamsConfigurator also supplies StreamsConfiguration POJO to runtimes and modules.
- *
- */
-public class StreamsConfigurator {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ComponentConfigurator.class);
-
- private static final ObjectMapper mapper = new ObjectMapper();
-
- /*
- Pull all configuration files from the classpath, system properties, and environment variables
- */
- private static Config config = ConfigFactory.load();
-
- public static Config getConfig() {
- return config.resolve();
- }
-
- public static void addConfig(Config newConfig) {
- config = newConfig.withFallback(config);
- }
-
- public static void setConfig(Config newConfig) {
- config = newConfig;
- }
-
- public static Config resolveConfig(String configUrl) throws MalformedURLException {
- URL url = new URL(configUrl);
- Config urlConfig = ConfigFactory.parseURL(url);
- urlConfig.resolve();
- config = urlConfig;
- return config;
- }
-
- public static StreamsConfiguration detectConfiguration() {
- return detectConfiguration(config);
- }
-
- public static StreamsConfiguration detectConfiguration(Config typesafeConfig) {
-
- StreamsConfiguration pojoConfig = null;
-
- try {
- pojoConfig = mapper.readValue(typesafeConfig.resolve().root().render(ConfigRenderOptions.concise()), StreamsConfiguration.class);
- } catch (Exception e) {
- e.printStackTrace();
- LOGGER.warn("Could not parse:", typesafeConfig);
- }
-
- return pojoConfig;
- }
-
- public static StreamsConfiguration mergeConfigurations(Config base, Config delta) {
-
- Config merged = delta.withFallback(base);
-
- StreamsConfiguration pojoConfig = null;
-
- try {
- pojoConfig = mapper.readValue(merged.resolve().root().render(ConfigRenderOptions.concise()), StreamsConfiguration.class);
- } catch (Exception e) {
- e.printStackTrace();
- LOGGER.warn("Failed to merge.");
- }
-
- return pojoConfig;
- }
-}
diff --git a/streams-config/src/main/jsonschema/ComponentConfiguration.json b/streams-config/src/main/jsonschema/ComponentConfiguration.json
deleted file mode 100644
index 4e015fe..0000000
--- a/streams-config/src/main/jsonschema/ComponentConfiguration.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.config.ComponentConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "inClasses": {
- "type" : "array",
- "items" : {
- "type" : "string"
- },
- "default": ["java.lang.String"]
- },
- "outClasses": {
- "type" : "array",
- "items" : {
- "type" : "string"
- },
- "default": ["java.lang.String"]
- }
- }
-}
\ No newline at end of file
diff --git a/streams-config/src/main/jsonschema/StreamsConfiguration.json b/streams-config/src/main/jsonschema/StreamsConfiguration.json
deleted file mode 100644
index eb24cdb..0000000
--- a/streams-config/src/main/jsonschema/StreamsConfiguration.json
+++ /dev/null
@@ -1,48 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.config.StreamsConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "batchFrequencyMs": {
- "type": "integer",
- "description": "Frequency of messaging between components"
- },
- "batchSize": {
- "type" : "integer",
- "description" : "Default max batch size for batch-based persist readers and writers"
- },
- "identifier": {
- "type" : "string",
- "description" : "Name of streams pipeline"
- },
- "parallelism": {
- "type" : "integer",
- "description" : "Default parallelism for processors"
- },
- "providerTimeoutMs": {
- "type": "integer",
- "description": "Kill the stream if all providers supply nothing for this long."
- },
- "queueSize": {
- "type" : "integer",
- "description" : "Size of inter-component buffers"
- },
- "shutdownCheckDelay": {
- "type": "integer",
- "description": "Initial delay before checking for shutdown"
- },
- "shutdownCheckInterval": {
- "type": "integer",
- "description": "Recurring delay between checking for shutdown"
- },
- "startedAt": {
- "type": "integer",
- "description": "Timestamp of beginning of stream execution"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-config/src/main/resources/reference.conf b/streams-config/src/main/resources/reference.conf
deleted file mode 100644
index 566d56c..0000000
--- a/streams-config/src/main/resources/reference.conf
+++ /dev/null
@@ -1,11 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-batchFrequencyMs = 5000
-batchSize = 1000
-identifier = "No-name Stream"
-parallelism = 1
-providerTimeoutMs = 1440000
-queueSize = 1000
-shutdownCheckDelay = 1000
-shutdownCheckInterval = 1000
-startedAt = -1
diff --git a/streams-config/src/site/markdown/index.md b/streams-config/src/site/markdown/index.md
deleted file mode 100644
index a262abd..0000000
--- a/streams-config/src/site/markdown/index.md
+++ /dev/null
@@ -1,8 +0,0 @@
-streams-config
-==============
-
-streams-config provides a centralized configuration system for streams.
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-config/src/test/java/org/apache/streams/config/test/ComponentConfiguratorTest.java b/streams-config/src/test/java/org/apache/streams/config/test/ComponentConfiguratorTest.java
deleted file mode 100644
index 80fa7de..0000000
--- a/streams-config/src/test/java/org/apache/streams/config/test/ComponentConfiguratorTest.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.config.test;
-
-import org.apache.streams.config.ComponentConfiguration;
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigRenderOptions;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-/**
-* Test for
-* @see org.apache.streams.config.ComponentConfigurator
-*/
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(StreamsConfigurator.class)
-public class ComponentConfiguratorTest {
-
- private static final ObjectMapper mapper = new ObjectMapper();
-
- @Test
- public void testDetectDefaults() throws Exception {
-
- Config config = ConfigFactory.load("componentTest");
-
- ComponentConfigurator<ComponentConfiguration> configurator = new ComponentConfigurator<>(ComponentConfiguration.class);
-
- ComponentConfiguration defaultPojo = configurator.detectConfiguration(config.getConfig("defaultComponent"));
-
- assert(defaultPojo != null);
-
- ComponentConfiguration configuredPojo = configurator.detectConfiguration(config.getConfig("configuredComponent"));
-
- assert(configuredPojo != null);
-
- Assert.assertEquals(configuredPojo,defaultPojo);
-
- }
-
- @Test
- public void testDetectConfigurationConfig() throws Exception {
-
- Config config = ConfigFactory.load("componentTest").getConfig("configuredComponent");
-
- ComponentConfigurator<ComponentConfiguration> configurator = new ComponentConfigurator<>(ComponentConfiguration.class);
-
- ComponentConfiguration testPojo = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), ComponentConfiguration.class);
-
- assert(testPojo != null);
-
- ComponentConfiguration configuredPojo = configurator.detectConfiguration(config);
-
- assert(configuredPojo != null);
-
- Assert.assertEquals(configuredPojo,testPojo);
-
- }
-
- @Test
- public void testDetectConfigurationString() throws Exception {
-
- Config config = ConfigFactory.load("componentTest");
-
- PowerMockito.mockStatic(StreamsConfigurator.class);
-
- PowerMockito.when(StreamsConfigurator.getConfig())
- .thenReturn(config);
-
- ComponentConfigurator<ComponentConfiguration> configurator = new ComponentConfigurator<>(ComponentConfiguration.class);
-
- ComponentConfiguration testPojo = mapper.readValue(config.root().get("configuredComponent").render(ConfigRenderOptions.concise()), ComponentConfiguration.class);
-
- assert(testPojo != null);
-
- ComponentConfiguration configuredPojo = configurator.detectConfiguration("configuredComponent");
-
- assert(configuredPojo != null);
-
- Assert.assertEquals(configuredPojo,testPojo);
- }
-
- @Test
- public void testDetectConfigurationConfigString() throws Exception {
-
- Config config = ConfigFactory.load("componentTest");
-
- ComponentConfigurator<ComponentConfiguration> configurator = new ComponentConfigurator<>(ComponentConfiguration.class);
-
- ComponentConfiguration testPojo = mapper.readValue(config.root().get("configuredComponent").render(ConfigRenderOptions.concise()), ComponentConfiguration.class);
-
-
- assert(testPojo != null);
-
- ComponentConfiguration configuredPojo = configurator.detectConfiguration(config, "configuredComponent");
-
- assert(configuredPojo != null);
-
- Assert.assertEquals(configuredPojo,testPojo);
- }
-}
\ No newline at end of file
diff --git a/streams-config/src/test/java/org/apache/streams/config/test/StreamsConfiguratorTest.java b/streams-config/src/test/java/org/apache/streams/config/test/StreamsConfiguratorTest.java
deleted file mode 100644
index a648d50..0000000
--- a/streams-config/src/test/java/org/apache/streams/config/test/StreamsConfiguratorTest.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.config.test;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigValue;
-import com.typesafe.config.ConfigValueFactory;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Map;
-
-/**
- * Test for {@link org.apache.streams.config.StreamsConfigurator}
- */
-public class StreamsConfiguratorTest {
-
- @Test
- public void testDetectConfiguration() throws Exception {
-
- Config config = ConfigFactory.load();
-
- Config detected = StreamsConfigurator.getConfig();
-
- Assert.assertEquals(config, detected);
-
- StreamsConfiguration defaultPojo = StreamsConfigurator.detectConfiguration();
-
- assert(defaultPojo != null);
-
- StreamsConfiguration configuredPojo = StreamsConfigurator.detectConfiguration(StreamsConfigurator.getConfig());
-
- assert(configuredPojo != null);
-
- Assert.assertEquals(configuredPojo, defaultPojo);
-
- }
-
- @Test
- public void testReference() throws Exception {
-
- Config defaultConfig = StreamsConfigurator.getConfig();
-
- assert( defaultConfig.getLong("batchFrequencyMs") > 0);
- assert( defaultConfig.getLong("batchSize") > 0);
- assert( defaultConfig.getLong("parallelism") > 0);
- assert( defaultConfig.getLong("providerTimeoutMs") > 0);
- assert( defaultConfig.getLong("queueSize") > 0);
-
- StreamsConfiguration defaultPojo = StreamsConfigurator.detectConfiguration();
-
- assert(defaultPojo != null);
-
- assert( defaultPojo.getBatchFrequencyMs() > 0);
- assert( defaultPojo.getBatchSize() > 0);
- assert( defaultPojo.getParallelism() > 0);
- assert( defaultPojo.getProviderTimeoutMs() > 0);
- assert( defaultPojo.getQueueSize() > 0);
-
- }
-
- @Test
- public void testOverride() throws Exception {
-
- Config overrides = ConfigFactory.empty()
- .withValue("parallelism", ConfigValueFactory.fromAnyRef(100l));
- StreamsConfigurator.addConfig(overrides);
-
- Config withOverride = StreamsConfigurator.getConfig();
-
- assert( withOverride.getLong("parallelism") == 100);
-
- StreamsConfiguration defaultPojo = StreamsConfigurator.detectConfiguration();
-
- assert( defaultPojo != null);
-
- assert( defaultPojo.getParallelism() == 100);
-
- }
-
- @Test
- public void testResolve() throws Exception {
-
- Config overrides = ConfigFactory.parseResourcesAnySyntax("testResolve.conf");
-
- StreamsConfigurator.addConfig(overrides);
-
- Config withOverride = StreamsConfigurator.getConfig();
-
- assert( withOverride.getString("message") != null);
-
- assert( withOverride.getConfig("evenmore").getString("message") != null);
-
- assert( withOverride.getString("samemessage") != null);
-
- StreamsConfiguration defaultPojo = StreamsConfigurator.detectConfiguration();
-
- assert( defaultPojo != null);
-
- assert( defaultPojo.getAdditionalProperties().containsKey("message"));
-
- assert( defaultPojo.getAdditionalProperties().containsKey("samemessage"));
-
- assert( defaultPojo.getAdditionalProperties().containsKey("evenmore"));
-
- assert( defaultPojo.getAdditionalProperties().get("evenmore") instanceof Map);
-
- }
-}
diff --git a/streams-config/src/test/resources/componentTest.conf b/streams-config/src/test/resources/componentTest.conf
deleted file mode 100644
index 83b1155..0000000
--- a/streams-config/src/test/resources/componentTest.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-defaultComponent {
-
-}
-configuredComponent {
- inClasses = [
- java.lang.String
- ]
- outClasses = [
- java.lang.String
- ]
-}
diff --git a/streams-config/src/test/resources/evenmore.conf b/streams-config/src/test/resources/evenmore.conf
deleted file mode 100644
index b9e1d79..0000000
--- a/streams-config/src/test/resources/evenmore.conf
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-message = helloagain
diff --git a/streams-config/src/test/resources/more.conf b/streams-config/src/test/resources/more.conf
deleted file mode 100644
index c4063f7..0000000
--- a/streams-config/src/test/resources/more.conf
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-message = hello
diff --git a/streams-config/src/test/resources/testResolve.conf b/streams-config/src/test/resources/testResolve.conf
deleted file mode 100644
index b1619ba..0000000
--- a/streams-config/src/test/resources/testResolve.conf
+++ /dev/null
@@ -1,3 +0,0 @@
-include "more.conf"
-evenmore = { include "evenmore.conf"}
-samemessage = ${evenmore.message}
diff --git a/streams-contrib/README.md b/streams-contrib/README.md
deleted file mode 100644
index 2b94646..0000000
--- a/streams-contrib/README.md
+++ /dev/null
@@ -1,11 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-streams-contrib
-==============
-
-These modules contain data schemas, API synchronizations, and ActivityStreams
-conversions unique to specific sites and software packages.
-
-Please help improve these packages, or create new ones!
diff --git a/streams-contrib/pom.xml b/streams-contrib/pom.xml
deleted file mode 100644
index 5355ee5..0000000
--- a/streams-contrib/pom.xml
+++ /dev/null
@@ -1,91 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>apache-streams</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-contrib</artifactId>
-
- <packaging>pom</packaging>
- <name>streams-contrib</name>
-
- <description>Contributed modules for communicating with services using non-Streams protocols and/or non-Activity data formats.</description>
-
- <properties>
-
- </properties>
-
- <modules>
- <module>streams-persist-cassandra</module>
- <module>streams-persist-console</module>
- <module>streams-persist-elasticsearch</module>
- <module>streams-persist-filebuffer</module>
- <module>streams-persist-hbase</module>
- <module>streams-persist-hdfs</module>
- <module>streams-persist-graph</module>
- <module>streams-persist-kafka</module>
- <module>streams-persist-mongo</module>
- <module>streams-persist-neo4j</module>
- <module>streams-persist-riak</module>
- <module>streams-amazon-aws</module>
- <module>streams-processor-jackson</module>
- <module>streams-processor-json</module>
- <module>streams-processor-urls</module>
- <module>streams-processor-peoplepattern</module>
- <module>streams-provider-facebook</module>
- <module>streams-provider-google</module>
- <module>streams-provider-instagram</module>
- <module>streams-provider-moreover</module>
- <module>streams-provider-twitter</module>
- <module>streams-provider-sysomos</module>
- <module>streams-provider-rss</module>
- <module>streams-processor-regex</module>
- <module>streams-provider-youtube</module>
- </modules>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
-</project>
diff --git a/streams-contrib/streams-amazon-aws/pom.xml b/streams-contrib/streams-amazon-aws/pom.xml
deleted file mode 100644
index 6ea14f8..0000000
--- a/streams-contrib/streams-amazon-aws/pom.xml
+++ /dev/null
@@ -1,80 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-amazon-aws</artifactId>
-
- <packaging>pom</packaging>
- <name>streams-amazon-aws</name>
-
- <description>Amazon AWS Modules</description>
-
- <properties>
- <amazonaws.version>1.11.184</amazonaws.version>
- </properties>
-
- <modules>
- <module>streams-persist-kinesis</module>
- <module>streams-persist-s3</module>
- </modules>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-kinesis</artifactId>
- <version>${amazonaws.version}</version>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
- <version>${amazonaws.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- </dependencyManagement>
-</project>
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/README.md b/streams-contrib/streams-amazon-aws/streams-persist-kinesis/README.md
deleted file mode 100644
index 395705a..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-persist-kinesis
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/pom.xml b/streams-contrib/streams-amazon-aws/streams-persist-kinesis/pom.xml
deleted file mode 100644
index d1c0b04..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/pom.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-amazon-aws</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-kinesis</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Amazon Kinesis Module</description>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-converters</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-kinesis</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.amazon.kinesis.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/java/org/apache/streams/amazon/kinesis/KinesisPersistReader.java b/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/java/org/apache/streams/amazon/kinesis/KinesisPersistReader.java
deleted file mode 100644
index 1ffe947..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/java/org/apache/streams/amazon/kinesis/KinesisPersistReader.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.amazon.kinesis;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.regions.Region;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.kinesis.AmazonKinesisClient;
-import com.amazonaws.services.kinesis.model.DescribeStreamResult;
-import com.amazonaws.services.kinesis.model.Shard;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import org.apache.commons.lang3.StringUtils;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-/**
- * KinesisPersistReader reads documents from kinesis.
- */
-public class KinesisPersistReader implements StreamsPersistReader, Serializable {
-
- public static final String STREAMS_ID = "KinesisPersistReader";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(KinesisPersistReader.class);
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = new ObjectMapper();
-
- private KinesisReaderConfiguration config;
-
- protected Long pollInterval = StreamsConfigurator.detectConfiguration().getBatchFrequencyMs();
-
- private List<String> streamNames;
-
- private ExecutorService executor;
-
- protected AmazonKinesisClient client;
-
- /**
- * KinesisPersistReader constructor - resolves KinesisReaderConfiguration from JVM 'kinesis'.
- */
- public KinesisPersistReader() {
- Config config = StreamsConfigurator.getConfig().getConfig("kinesis");
- this.config = new ComponentConfigurator<>(KinesisReaderConfiguration.class).detectConfiguration(config);
- this.persistQueue = new ConcurrentLinkedQueue<>();
- }
-
- /**
- * KinesisPersistReader constructor - uses provided KinesisReaderConfiguration.
- */
- public KinesisPersistReader(KinesisReaderConfiguration config) {
- this.config = config;
- this.persistQueue = new ConcurrentLinkedQueue<>();
- }
-
- public void setConfig(KinesisReaderConfiguration config) {
- this.config = config;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
-
- this.streamNames = this.config.getStreams();
-
- for (final String stream : streamNames) {
-
- DescribeStreamResult describeStreamResult = client.describeStream(stream);
-
- if( "ACTIVE".equals(describeStreamResult.getStreamDescription().getStreamStatus())) {
-
- List<Shard> shardList = describeStreamResult.getStreamDescription().getShards();
-
- for( Shard shard : shardList ) {
- executor.submit(new KinesisPersistReaderTask(this, stream, shard.getShardId()));
- }
- }
-
- }
-
- }
-
- @Override
- public StreamsResultSet readAll() {
- return readCurrent();
- }
-
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
- synchronized( KinesisPersistReader.class ) {
- current = new StreamsResultSet(new ConcurrentLinkedQueue<>(persistQueue));
- persistQueue.clear();
- }
- return current;
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger bigInteger) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime dateTime, DateTime dateTime2) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return !executor.isShutdown() && !executor.isTerminated();
- }
-
- @Override
- public void prepare(Object configurationObject) {
- // Connect to Kinesis
- synchronized (this) {
- // Create the credentials Object
- AWSCredentials credentials = new BasicAWSCredentials(config.getKey(), config.getSecretKey());
-
- ClientConfiguration clientConfig = new ClientConfiguration();
- clientConfig.setProtocol(Protocol.valueOf(config.getProtocol().toString()));
-
- this.client = new AmazonKinesisClient(credentials, clientConfig);
- if (StringUtils.isNotEmpty(config.getRegion()))
- this.client.setRegion(Region.getRegion(Regions.fromName(config.getRegion())));
- }
- streamNames = this.config.getStreams();
- executor = Executors.newFixedThreadPool(streamNames.size());
- }
-
- @Override
- public void cleanUp() {
-
- while( !executor.isTerminated()) {
- try {
- executor.awaitTermination(5, TimeUnit.SECONDS);
- } catch (InterruptedException ignored) {}
- }
- }
-}
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/java/org/apache/streams/amazon/kinesis/KinesisPersistReaderTask.java b/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/java/org/apache/streams/amazon/kinesis/KinesisPersistReaderTask.java
deleted file mode 100644
index eb21c00..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/java/org/apache/streams/amazon/kinesis/KinesisPersistReaderTask.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.amazon.kinesis;
-
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-
-import com.amazonaws.services.kinesis.model.GetRecordsRequest;
-import com.amazonaws.services.kinesis.model.GetRecordsResult;
-import com.amazonaws.services.kinesis.model.GetShardIteratorRequest;
-import com.amazonaws.services.kinesis.model.GetShardIteratorResult;
-import com.amazonaws.services.kinesis.model.Record;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.math.BigInteger;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * KinesisPersistReaderTask reads documents from kinesis on behalf of
- * @see {@link KinesisPersistReader}.
- */
-public class KinesisPersistReaderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(KinesisPersistReaderTask.class);
-
- private KinesisPersistReader reader;
- private String streamName;
- private String shardId;
-
- private String shardIteratorId;
-
- private Long pollInterval = StreamsConfigurator.detectConfiguration().getBatchFrequencyMs();
-
- /**
- * KinesisPersistReaderTask constructor.
- */
- public KinesisPersistReaderTask(KinesisPersistReader reader, String streamName, String shardId) {
- this.reader = reader;
- this.streamName = streamName;
- this.shardId = shardId;
- }
-
- @Override
- public void run() {
-
- GetShardIteratorRequest shardIteratorRequest = new GetShardIteratorRequest()
- .withStreamName(this.streamName)
- .withShardId(shardId)
- .withShardIteratorType("TRIM_HORIZON");
-
- GetShardIteratorResult shardIteratorResult = reader.client.getShardIterator(shardIteratorRequest);
-
- shardIteratorId = shardIteratorResult.getShardIterator();
-
- Map<String,Object> metadata = new HashMap<>();
- metadata.put("streamName", streamName);
- metadata.put("shardId", shardId);
-
- while (true) {
-
- GetRecordsRequest recordsRequest = new GetRecordsRequest()
- .withShardIterator(shardIteratorId);
-
- GetRecordsResult recordsResult = reader.client.getRecords(recordsRequest);
-
- LOGGER.info("{} records {} millis behind {}:{}:{} ", recordsResult.getRecords().size(), recordsResult.getMillisBehindLatest(), streamName, shardId, shardIteratorId);
-
- shardIteratorId = recordsResult.getNextShardIterator();
-
- List<Record> recordList = recordsResult.getRecords();
-
- for (Record record : recordList) {
- try {
- byte[] byteArray = record.getData().array();
- //byte[] decoded = Base64.decode(byteArray);
- String message = new String(byteArray, Charset.forName("UTF-8"));
- reader.persistQueue.add(
- new StreamsDatum(
- message,
- record.getPartitionKey(),
- new DateTime(),
- new BigInteger(record.getSequenceNumber()),
- metadata));
- } catch ( Exception ex ) {
- LOGGER.warn("Exception processing record {}: {}", record, ex);
- }
- }
- try {
- Thread.sleep(reader.pollInterval);
- } catch (InterruptedException ex) {
- LOGGER.trace("InterruptedException", ex);
- }
- }
-
- }
-
-}
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/java/org/apache/streams/amazon/kinesis/KinesisPersistWriter.java b/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/java/org/apache/streams/amazon/kinesis/KinesisPersistWriter.java
deleted file mode 100644
index 6a16e89..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/java/org/apache/streams/amazon/kinesis/KinesisPersistWriter.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.amazon.kinesis;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.converter.TypeConverterUtil;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.regions.Region;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.kinesis.AmazonKinesisClient;
-import com.amazonaws.services.kinesis.model.PutRecordRequest;
-import com.amazonaws.services.kinesis.model.PutRecordResult;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-/**
- * KinesisPersistWriter writes documents to kinesis.
- */
-public class KinesisPersistWriter implements StreamsPersistWriter {
-
- public static final String STREAMS_ID = "KinesisPersistWriter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(KinesisPersistWriter.class);
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = new ObjectMapper();
-
- private KinesisWriterConfiguration config;
-
- private List<String> streamName;
-
- private ExecutorService executor;
-
- protected AmazonKinesisClient client;
-
- /**
- * KinesisPersistWriter constructor - resolves KinesisWriterConfiguration from JVM 'kinesis'.
- */
- public KinesisPersistWriter() {
- Config config = StreamsConfigurator.getConfig().getConfig("kinesis");
- this.config = new ComponentConfigurator<>(KinesisWriterConfiguration.class).detectConfiguration(config);
- this.persistQueue = new ConcurrentLinkedQueue<>();
- }
-
- /**
- * KinesisPersistWriter constructor - uses provided KinesisWriterConfiguration.
- */
- public KinesisPersistWriter(KinesisWriterConfiguration config) {
- this.config = config;
- this.persistQueue = new ConcurrentLinkedQueue<>();
- }
-
- public void setConfig(KinesisWriterConfiguration config) {
- this.config = config;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void write(StreamsDatum entry) {
-
- String document = (String) TypeConverterUtil.getInstance().convert(entry.getDocument(), String.class);
-
- PutRecordRequest putRecordRequest = new PutRecordRequest()
- .withStreamName(config.getStream())
- .withPartitionKey(entry.getId())
- .withData(ByteBuffer.wrap(document.getBytes()));
-
- PutRecordResult putRecordResult = client.putRecord(putRecordRequest);
-
- entry.setSequenceid(new BigInteger(putRecordResult.getSequenceNumber()));
-
- LOGGER.debug("Wrote {}", entry);
- }
-
- @Override
- public void prepare(Object configurationObject) {
- // Connect to Kinesis
- synchronized (this) {
- // Create the credentials Object
- AWSCredentials credentials = new BasicAWSCredentials(config.getKey(), config.getSecretKey());
-
- ClientConfiguration clientConfig = new ClientConfiguration();
- clientConfig.setProtocol(Protocol.valueOf(config.getProtocol().toString()));
-
- this.client = new AmazonKinesisClient(credentials, clientConfig);
- if (StringUtils.isNotEmpty(config.getRegion())) {
- this.client.setRegion(Region.getRegion(Regions.fromName(config.getRegion())));
- }
- }
- executor = Executors.newSingleThreadExecutor();
-
- }
-
- @Override
- public void cleanUp() {
- try {
- executor.awaitTermination(5, TimeUnit.SECONDS);
- } catch (InterruptedException ex) {
- LOGGER.debug("Interrupted! ", ex);
- }
- }
-}
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/jsonschema/org/apache/streams/amazon/kinesis/KinesisConfiguration.json b/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/jsonschema/org/apache/streams/amazon/kinesis/KinesisConfiguration.json
deleted file mode 100644
index 05c7606..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/jsonschema/org/apache/streams/amazon/kinesis/KinesisConfiguration.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.amazon.kinesis.KinesisConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "key": {
- "type": "string",
- "description": "Your Amazon Key",
- "required": true
- },
- "secretKey": {
- "type": "string",
- "description": "Your Amazon Secret Key",
- "required": true
- },
- "protocol": {
- "type": "string",
- "description": "Whether you are using HTTP or HTTPS",
- "enum": ["HTTP", "HTTPS"],
- "default": "HTTPS"
- },
- "region": {
- "type": "string",
- "description": "The AWS region where your bucket resides",
- "required": false
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/jsonschema/org/apache/streams/amazon/kinesis/KinesisReaderConfiguration.json b/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/jsonschema/org/apache/streams/amazon/kinesis/KinesisReaderConfiguration.json
deleted file mode 100644
index bf58892..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/jsonschema/org/apache/streams/amazon/kinesis/KinesisReaderConfiguration.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.amazon.kinesis.KinesisReaderConfiguration",
- "extends": {"$ref":"KinesisConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "streams": {
- "type": "array",
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/jsonschema/org/apache/streams/amazon/kinesis/KinesisWriterConfiguration.json b/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/jsonschema/org/apache/streams/amazon/kinesis/KinesisWriterConfiguration.json
deleted file mode 100644
index 2fcf059..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/jsonschema/org/apache/streams/amazon/kinesis/KinesisWriterConfiguration.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.amazon.kinesis.KinesisWriterConfiguration",
- "extends": {"$ref":"KinesisConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "stream": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/resources/components.dot b/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/resources/components.dot
deleted file mode 100644
index 072751d..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/resources/components.dot
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- processors [label="processors", shape="circle"];
-
- subgraph cluster_persisters {
- label="persisters";
- persisters_kinesis_reader [label="KinesisPersistReader"]
- persisters_kinesis_writer [label="KinesisPersistWriter"]
- }
-
- subgraph cluster_queues {
- label="queues";
- kinesis [label="kinesis"]
- }
-
- generators -> providers
- providers -> processors
- processors -> persisters_kinesis_writer [label="StreamsDatum"]
- persisters_kinesis_reader -> processors [label="StreamsDatum[String]"]
- persisters_kinesis_writer -> kinesis
- kinesis -> persisters_kinesis_reader
-}
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/resources/kinesis-read.conf b/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/resources/kinesis-read.conf
deleted file mode 100644
index 8026abd..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/resources/kinesis-read.conf
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-kinesis: {
- key: ""
- secretKey: ""
- protocol: "HTTPS"
- region: "us-east-1"
- streams = [
- "topic1"
- "topic2"
- ]
-}
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/resources/reference.conf b/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/resources/reference.conf
deleted file mode 100644
index fc1909b..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/main/resources/reference.conf
+++ /dev/null
@@ -1,5 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-kinesis {
- protocol = HTTPS
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/site/markdown/index.md b/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/site/markdown/index.md
deleted file mode 100644
index 32f5781..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-kinesis/src/site/markdown/index.md
+++ /dev/null
@@ -1,23 +0,0 @@
-streams-persist-kinesis
-==============
-
-Read/Write documents to/from Kinesis.
-
-## Configuration
-
-| Schema |
-|--------|
-| [KinesisConfiguration.json](org/apache/streams/amazon/kinesis/KinesisConfiguration.json "KinesisConfiguration.json") [KinesisConfiguration.html](apidocs/org/apache/streams/amazon/kinesis/KinesisConfiguration.html "javadoc") |
-
-## Components
-
-
-
-| Class | Configuration | Example Configuration(s) |
-|-------|---------------|--------------------------|
-| KinesisPersistReader [KinesisPersistReader.html](apidocs/org/apache/streams/amazon/kinesis/KinesisPersistReader.html "javadoc") | [KinesisReaderConfiguration.json](org/apache/streams/amazon/kinesis/KinesisReaderConfiguration.json "KinesisReaderConfiguration.json") [KinesisReaderConfiguration.html](apidocs/org/apache/streams/amazon/kinesis/KinesisReaderConfiguration.html "javadoc") | [kinesis-read.conf](kinesis-read.conf "kinesis-read.conf") |
-| KinesisPersistWriter [KinesisPersistWriter.html](apidocs/org/apache/streams/amazon/kinesis/KinesisPersistWriter "javadoc") | [KinesisWriterConfiguration.json](org/apache/streams/amazon/kinesis/KinesisWriterConfiguration.json "KinesisWriterConfiguration.json") [KinesisWriterConfiguration.html](apidocs/org/apache/streams/amazon/kinesis/KinesisWriterConfiguration.html "javadoc") | [kinesis-write.conf](kinesis-write.conf "kinesis-write.conf") |
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/README.md b/streams-contrib/streams-amazon-aws/streams-persist-s3/README.md
deleted file mode 100644
index 1d71117..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-persist-s3
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/pom.xml b/streams-contrib/streams-amazon-aws/streams-persist-s3/pom.xml
deleted file mode 100644
index 0922fc9..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/pom.xml
+++ /dev/null
@@ -1,144 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-amazon-aws</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-s3</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Amazon S3 Module</description>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-converters</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-persist-hdfs</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.s3.pojo</targetPackage>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-converters</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-converters</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </plugin>
-
- </plugins>
- </build>
-</project>
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3ObjectInputStreamWrapper.java b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3ObjectInputStreamWrapper.java
deleted file mode 100644
index e70c361..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3ObjectInputStreamWrapper.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.s3;
-
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectInputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * There is a nuance associated with reading portions of files in S3. Everything occurs over
- * an Apache HTTP client object. Apache and therefore Amazon defaults to re-using the stream.
- * As a result, if you only intend read a small portion of the file. You must first "abort" the
- * stream, then close the 'inputStream'. Otherwise, Apache will exhaust the entire stream
- * and transfer the entire file. If you are only reading the first 50 lines of a 5,000,000 line file
- * this becomes problematic.
- *
- * <p/>
- * This class operates as a wrapper to fix the aforementioned nuances.
- *
- * <p/>
- * Reference:
- * http://stackoverflow.com/questions/17782937/connectionpooltimeoutexception-when-iterating-objects-in-s3
- */
-public class S3ObjectInputStreamWrapper extends InputStream {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(S3ObjectInputStreamWrapper.class);
-
- private final S3Object s3Object;
- private final S3ObjectInputStream is;
- private boolean isClosed = false;
-
- /**
- * Create an input stream safely.
- * @param s3Object s3Object
- */
- public S3ObjectInputStreamWrapper(S3Object s3Object) {
- this.s3Object = s3Object;
- this.is = this.s3Object.getObjectContent();
- }
-
- public int hashCode() {
- return this.is.hashCode();
- }
-
- public boolean equals(Object obj) {
- return this.is.equals(obj);
- }
-
- public String toString() {
- return this.is.toString();
- }
-
- public int read() throws IOException {
- return this.is.read();
- }
-
- public int read(byte[] byt) throws IOException {
- return this.is.read(byt);
- }
-
- public int read(byte[] byt, int off, int len) throws IOException {
- return this.is.read(byt, off, len);
- }
-
- public long skip(long skip) throws IOException {
- return this.is.skip(skip);
- }
-
- public int available() throws IOException {
- return this.is.available();
- }
-
- public boolean markSupported() {
- return this.is.markSupported();
- }
-
- public synchronized void mark(int readlimit) {
- this.is.mark(readlimit);
- }
-
- public synchronized void reset() throws IOException {
- this.is.reset();
- }
-
- public void close() throws IOException {
- ensureEverythingIsReleased();
- }
-
- /**
- * ensureEverythingIsReleased as part of close process.
- */
- public void ensureEverythingIsReleased() {
- if (this.isClosed) {
- return;
- }
-
- try {
- // ensure that the S3 Object is closed properly.
- this.s3Object.close();
- } catch (Throwable ex) {
- LOGGER.warn("Problem Closing the S3Object[{}]: {}", s3Object.getKey(), ex.getMessage());
- }
-
-
- try {
- // Abort the stream
- this.is.abort();
- } catch (Throwable ex) {
- LOGGER.warn("Problem Aborting S3Object[{}]: {}", s3Object.getKey(), ex.getMessage());
- }
-
- // close the input Stream Safely
- closeSafely(this.is);
-
- // This corrects the issue with Open HTTP connections
- closeSafely(this.s3Object);
- this.isClosed = true;
- }
-
- private static void closeSafely(Closeable is) {
- try {
- if (is != null) {
- is.close();
- }
- } catch (Exception ex) {
- ex.printStackTrace();
- LOGGER.warn("S3InputStreamWrapper: Issue Closing Closeable - {}", ex.getMessage());
- }
- }
-
- protected void finalize() throws Throwable {
- try {
- // If there is an accidental leak where the user did not close, call this on the classes destructor
- ensureEverythingIsReleased();
- super.finalize();
- } catch (Exception ex) {
- // this should never be called, just being very cautious
- LOGGER.warn("S3InputStreamWrapper: Issue Releasing Connections on Finalize - {}", ex.getMessage());
- }
- }
-
-}
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3OutputStreamWrapper.java b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3OutputStreamWrapper.java
deleted file mode 100644
index 87bd8c8..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3OutputStreamWrapper.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.s3;
-
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.transfer.TransferManager;
-import com.amazonaws.services.s3.transfer.Upload;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Map;
-
-/**
- * This class uses ByteArrayOutputStreams to ensure files are written to S3 properly. The stream is written to the
- * in memory ByteArrayOutPutStream before it is finally written to Amazon S3. The size the file is allowed to become
- * is directly controlled by the S3PersistWriter.
- */
-public class S3OutputStreamWrapper extends OutputStream {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(S3OutputStreamWrapper.class);
-
- private final AmazonS3Client amazonS3Client;
- private final String bucketName;
- private final String path;
- private final String fileName;
- private ByteArrayOutputStream outputStream;
- private final Map<String, String> metaData;
- private boolean isClosed = false;
-
- /**
- * Create an OutputStream Wrapper
- * @param amazonS3Client
- * The Amazon S3 Client which will be handling the file
- * @param bucketName
- * The Bucket Name you are wishing to write to.
- * @param path
- * The path where the object will live
- * @param fileName
- * The fileName you ware wishing to write.
- * @param metaData
- * Any meta data that is to be written along with the object
- * @throws IOException
- * If there is an issue creating the stream, this
- */
- public S3OutputStreamWrapper(AmazonS3Client amazonS3Client, String bucketName, String path, String fileName, Map<String, String> metaData) throws IOException {
- this.amazonS3Client = amazonS3Client;
- this.bucketName = bucketName;
- this.path = path;
- this.fileName = fileName;
- this.metaData = metaData;
- this.outputStream = new ByteArrayOutputStream();
- }
-
- public void write(int byt) throws IOException {
- this.outputStream.write(byt);
- }
-
- public void write(byte[] byt) throws IOException {
- this.outputStream.write(byt);
- }
-
- public void write(byte[] byt, int off, int len) throws IOException {
- this.outputStream.write(byt, off, len);
- }
-
- public void flush() throws IOException {
- this.outputStream.flush();
- }
-
- /**
- * Whenever the output stream is closed we are going to kick the ByteArrayOutputStream off to Amazon S3.
- * @throws IOException
- * Exception thrown from the FileOutputStream
- */
- public void close() throws IOException {
- if (!isClosed) {
- try {
- this.addFile();
- this.outputStream.close();
- this.outputStream = null;
- } catch (Exception ex) {
- ex.printStackTrace();
- LOGGER.warn("There was an error adding the temporaryFile to S3");
- } finally {
- // we are done here.
- this.isClosed = true;
- }
- }
- }
-
- private void addFile() throws Exception {
-
- InputStream is = new ByteArrayInputStream(this.outputStream.toByteArray());
- int contentLength = outputStream.size();
-
- TransferManager transferManager = new TransferManager(amazonS3Client);
- ObjectMetadata metadata = new ObjectMetadata();
- metadata.setExpirationTime(DateTime.now().plusDays(365 * 3).toDate());
- metadata.setContentLength(contentLength);
-
- metadata.addUserMetadata("writer", "org.apache.streams");
-
- for (String s : metaData.keySet()) {
- metadata.addUserMetadata(s, metaData.get(s));
- }
-
- String fileNameToWrite = path + fileName;
- Upload upload = transferManager.upload(bucketName, fileNameToWrite, is, metadata);
- try {
- upload.waitForUploadResult();
-
- is.close();
- transferManager.shutdownNow(false);
- LOGGER.info("S3 File Close[{} kb] - {}", contentLength / 1024, path + fileName);
- } catch (Exception ignored) {
- LOGGER.trace("Ignoring", ignored);
- }
-
-
- }
-
-
-}
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
deleted file mode 100644
index 3a2cf3b..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.s3;
-
-import org.apache.streams.converter.LineReadWriteUtil;
-import org.apache.streams.core.DatumStatusCountable;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.regions.Region;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.S3ClientOptions;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Queues;
-import org.apache.commons.lang3.StringUtils;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-
-/**
- * S3PersistReader reads documents from s3.
- */
-public class S3PersistReader implements StreamsPersistReader, DatumStatusCountable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(S3PersistReader.class);
- public static final String STREAMS_ID = "S3PersistReader";
- protected static final char DELIMITER = '\t';
-
- private S3ReaderConfiguration s3ReaderConfiguration;
- private AmazonS3Client amazonS3Client;
- private ObjectMapper mapper = new ObjectMapper();
- protected LineReadWriteUtil lineReaderUtil;
- private Collection<String> files;
- private ExecutorService executor;
- protected volatile Queue<StreamsDatum> persistQueue;
-
- protected DatumStatusCounter countersTotal = new DatumStatusCounter();
- protected DatumStatusCounter countersCurrent = new DatumStatusCounter();
- private Future<?> task;
-
- public AmazonS3Client getAmazonS3Client() {
- return this.amazonS3Client;
- }
-
- public S3ReaderConfiguration getS3ReaderConfiguration() {
- return this.s3ReaderConfiguration;
- }
-
- public String getBucketName() {
- return this.s3ReaderConfiguration.getBucket();
- }
-
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return !task.isDone() && !task.isCancelled();
- }
-
- public DatumStatusCounter getDatumStatusCounter() {
- return countersTotal;
- }
-
- public Collection<String> getFiles() {
- return this.files;
- }
-
- public S3PersistReader(S3ReaderConfiguration s3ReaderConfiguration) {
- this.s3ReaderConfiguration = s3ReaderConfiguration;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- lineReaderUtil = LineReadWriteUtil.getInstance(s3ReaderConfiguration);
- // Connect to S3
- synchronized (this) {
- // Create the credentials Object
- AWSCredentials credentials = new BasicAWSCredentials(s3ReaderConfiguration.getKey(), s3ReaderConfiguration.getSecretKey());
-
- ClientConfiguration clientConfig = new ClientConfiguration();
- clientConfig.setProtocol(Protocol.valueOf(s3ReaderConfiguration.getProtocol().toString()));
-
- // We do not want path style access
- S3ClientOptions clientOptions = new S3ClientOptions();
- clientOptions.setPathStyleAccess(false);
-
- this.amazonS3Client = new AmazonS3Client(credentials, clientConfig);
- if (StringUtils.isNotEmpty(s3ReaderConfiguration.getRegion())) {
- this.amazonS3Client.setRegion(Region.getRegion(Regions.fromName(s3ReaderConfiguration.getRegion())));
- }
- this.amazonS3Client.setS3ClientOptions(clientOptions);
- }
-
- final ListObjectsRequest request = new ListObjectsRequest()
- .withBucketName(this.s3ReaderConfiguration.getBucket())
- .withPrefix(s3ReaderConfiguration.getReaderPath())
- .withMaxKeys(500);
-
-
- ObjectListing listing = this.amazonS3Client.listObjects(request);
-
- this.files = new ArrayList<>();
-
- /*
- * If you can list files that are in this path, then you must be dealing with a directory
- * if you cannot list files that are in this path, then you are most likely dealing with
- * a simple file.
- */
- boolean hasCommonPrefixes = listing.getCommonPrefixes().size() > 0;
- boolean hasObjectSummaries = listing.getObjectSummaries().size() > 0;
-
- if (hasCommonPrefixes || hasObjectSummaries) {
- // Handle the 'directory' use case
- do {
- if (hasCommonPrefixes) {
- for (String file : listing.getCommonPrefixes()) {
- this.files.add(file);
- }
- } else {
- for (final S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
- this.files.add(objectSummary.getKey());
- }
- }
-
- // get the next batch.
- listing = this.amazonS3Client.listNextBatchOfObjects(listing);
- }
- while (listing.isTruncated());
- } else {
- // handle the single file use-case
- this.files.add(s3ReaderConfiguration.getReaderPath());
- }
-
- if (this.files.size() <= 0) {
- LOGGER.error("There are no files to read");
- }
-
- this.persistQueue = Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
- this.executor = Executors.newSingleThreadExecutor();
- }
-
- public void cleanUp() {
- // no Op
- }
-
- public StreamsResultSet readAll() {
- startStream();
- return new StreamsResultSet(persistQueue);
- }
-
- public void startStream() {
- LOGGER.debug("startStream");
- task = executor.submit(new S3PersistReaderTask(this));
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- synchronized ( S3PersistReader.class ) {
- current = new StreamsResultSet(new ConcurrentLinkedQueue<>(persistQueue));
- current.setCounter(new DatumStatusCounter());
- current.getCounter().add(countersCurrent);
- countersTotal.add(countersCurrent);
- countersCurrent = new DatumStatusCounter();
- persistQueue.clear();
- }
- return current;
- }
-
-}
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReaderTask.java b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReaderTask.java
deleted file mode 100644
index 775e7e0..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReaderTask.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.s3;
-
-import org.apache.streams.core.DatumStatus;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.util.ComponentUtils;
-
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.Closeable;
-import java.io.InputStreamReader;
-
-/**
- * S3PersistReaderTask reads documents from s3 on behalf of
- * @see org.apache.streams.s3.S3PersistReader
- */
-public class S3PersistReaderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(S3PersistReaderTask.class);
-
- private S3PersistReader reader;
-
- public S3PersistReaderTask(S3PersistReader reader) {
- this.reader = reader;
- }
-
- @Override
- public void run() {
-
- for (String file : reader.getFiles()) {
-
- // Create our buffered reader
- S3ObjectInputStreamWrapper is = new S3ObjectInputStreamWrapper(reader.getAmazonS3Client().getObject(reader.getBucketName(), file));
- BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(is));
- LOGGER.info("Reading: {} ", file);
-
- String line;
- try {
- while ((line = bufferedReader.readLine()) != null) {
- if (StringUtils.isNotEmpty(line) ) {
- reader.countersCurrent.incrementAttempt();
- StreamsDatum entry = reader.lineReaderUtil.processLine(line);
- ComponentUtils.offerUntilSuccess(entry, reader.persistQueue);
- reader.countersCurrent.incrementStatus(DatumStatus.SUCCESS);
- }
- }
- } catch (Exception ex) {
- ex.printStackTrace();
- LOGGER.warn(ex.getMessage());
- reader.countersCurrent.incrementStatus(DatumStatus.FAIL);
- }
-
- LOGGER.info("Completed: " + file);
-
- try {
- closeSafely(file, is);
- } catch (Exception ex) {
- LOGGER.error(ex.getMessage());
- }
- }
- }
-
- private static void closeSafely(String file, Closeable closeable) {
- try {
- closeable.close();
- } catch (Exception ex) {
- LOGGER.error("There was an issue closing file: {}", file);
- }
- }
-}
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistWriter.java b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistWriter.java
deleted file mode 100644
index 97dfa56..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistWriter.java
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.s3;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.converter.LineReadWriteUtil;
-import org.apache.streams.core.DatumStatus;
-import org.apache.streams.core.DatumStatusCountable;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.regions.Region;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.S3ClientOptions;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Flushable;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * S3PersistWriter writes documents to s3.
- */
-public class S3PersistWriter implements StreamsPersistWriter, DatumStatusCountable {
-
- public static final String STREAMS_ID = "S3PersistWriter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(S3PersistWriter.class);
-
- private static final char DELIMITER = '\t';
-
- private ObjectMapper objectMapper;
- private AmazonS3Client amazonS3Client;
- private S3WriterConfiguration s3WriterConfiguration;
- private final List<String> writtenFiles = new ArrayList<>();
- protected LineReadWriteUtil lineWriterUtil;
-
- private final AtomicLong totalBytesWritten = new AtomicLong();
- private AtomicLong bytesWrittenThisFile = new AtomicLong();
-
- private final AtomicInteger totalRecordsWritten = new AtomicInteger();
- private AtomicInteger fileLineCounter = new AtomicInteger();
-
- private static Map<String, String> objectMetaData = new HashMap<>();
-
- static {
- objectMetaData.put("line[0]", "id");
- objectMetaData.put("line[1]", "timeStamp");
- objectMetaData.put("line[2]", "metaData");
- objectMetaData.put("line[3]", "document");
- }
-
- private OutputStreamWriter currentWriter = null;
-
- public AmazonS3Client getAmazonS3Client() {
- return this.amazonS3Client;
- }
-
- public S3WriterConfiguration getS3WriterConfiguration() {
- return this.s3WriterConfiguration;
- }
-
- public List<String> getWrittenFiles() {
- return this.writtenFiles;
- }
-
- public Map<String, String> getObjectMetaData() {
- return objectMetaData;
- }
-
- public ObjectMapper getObjectMapper() {
- return this.objectMapper;
- }
-
- public void setObjectMapper(ObjectMapper mapper) {
- this.objectMapper = mapper;
- }
-
- public void setObjectMetaData(Map<String, String> val) {
- objectMetaData = val;
- }
-
- public S3PersistWriter() {
- this(new ComponentConfigurator<>(S3WriterConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig().getConfig("s3")));
- }
-
- public S3PersistWriter(S3WriterConfiguration s3WriterConfiguration) {
- this.s3WriterConfiguration = s3WriterConfiguration;
- }
-
- /**
- * Instantiator with a pre-existing amazonS3Client, this is used to help with re-use.
- * @param amazonS3Client
- * If you have an existing amazonS3Client, it wont' bother to create another one
- * @param s3WriterConfiguration
- * Configuration of the write paths and instructions are still required.
- */
- public S3PersistWriter(AmazonS3Client amazonS3Client, S3WriterConfiguration s3WriterConfiguration) {
- this.amazonS3Client = amazonS3Client;
- this.s3WriterConfiguration = s3WriterConfiguration;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void write(StreamsDatum streamsDatum) {
-
- synchronized (this) {
- // Check to see if we need to reset the file that we are currently working with
- if (this.currentWriter == null || ( this.bytesWrittenThisFile.get() >= (this.s3WriterConfiguration.getMaxFileSize() * 1024 * 1024))) {
- try {
- LOGGER.info("Resetting the file");
- this.currentWriter = resetFile();
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
-
- String line = lineWriterUtil.convertResultToString(streamsDatum);
-
- try {
- this.currentWriter.write(line);
- } catch (IOException ex) {
- ex.printStackTrace();
- }
-
- // add the bytes we've written
- int recordSize = line.getBytes().length;
- this.totalBytesWritten.addAndGet(recordSize);
- this.bytesWrittenThisFile.addAndGet(recordSize);
-
- // increment the record count
- this.totalRecordsWritten.incrementAndGet();
- this.fileLineCounter.incrementAndGet();
- }
-
- }
-
- /**
- * Reset File when it's time to create a new file.
- * @return OutputStreamWriter
- * @throws Exception Exception
- */
- public synchronized OutputStreamWriter resetFile() throws Exception {
- // this will keep it thread safe, so we don't create too many files
- if (this.fileLineCounter.get() == 0 && this.currentWriter != null) {
- return this.currentWriter;
- }
-
- closeAndDestroyWriter();
-
- // Create the path for where the file is going to live.
- try {
- // generate a file name
- String fileName = this.s3WriterConfiguration.getWriterFilePrefix()
- + (this.s3WriterConfiguration.getChunk() ? "/" : "-")
- + new Date().getTime()
- + ".tsv";
-
- // create the output stream
- OutputStream outputStream = new S3OutputStreamWrapper(this.amazonS3Client,
- this.s3WriterConfiguration.getBucket(),
- this.s3WriterConfiguration.getWriterPath(),
- fileName,
- objectMetaData);
-
- // reset the counter
- this.fileLineCounter = new AtomicInteger();
- this.bytesWrittenThisFile = new AtomicLong();
-
- // add this to the list of written files
- writtenFiles.add(this.s3WriterConfiguration.getWriterPath() + fileName);
-
- // Log that we are creating this file
- LOGGER.info("File Created: Bucket[{}] - {}", this.s3WriterConfiguration.getBucket(), this.s3WriterConfiguration.getWriterPath() + fileName);
-
- // return the output stream
- return new OutputStreamWriter(outputStream);
- } catch (Exception ex) {
- LOGGER.error(ex.getMessage());
- throw ex;
- }
- }
-
- private synchronized void closeAndDestroyWriter() {
- // if there is a current writer, we must close it first.
- if (this.currentWriter != null) {
- this.safeFlush(this.currentWriter);
- this.closeSafely(this.currentWriter);
- this.currentWriter = null;
-
- // Logging of information to alert the user to the activities of this class
- LOGGER.debug("File Closed: Records[{}] Bytes[{}] {} ", this.fileLineCounter.get(), this.bytesWrittenThisFile.get(), this.writtenFiles.get(this.writtenFiles.size() - 1));
- }
- }
-
- private synchronized void closeSafely(Writer writer) {
- if (writer != null) {
- try {
- writer.flush();
- writer.close();
- } catch (Exception ex) {
- LOGGER.trace("closeSafely", ex);
- }
- LOGGER.debug("File Closed");
- }
- }
-
- private void safeFlush(Flushable flushable) {
- // This is wrapped with a ByteArrayOutputStream, so this is really safe.
- if (flushable != null) {
- try {
- flushable.flush();
- } catch (IOException ex) {
- LOGGER.trace("safeFlush", ex);
- }
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- lineWriterUtil = LineReadWriteUtil.getInstance(s3WriterConfiguration);
-
- // Connect to S3
- synchronized (this) {
-
- try {
- // if the user has chosen to not set the object mapper, then set a default object mapper for them.
- if (this.objectMapper == null) {
- this.objectMapper = StreamsJacksonMapper.getInstance();
- }
-
- // Create the credentials Object
- if (this.amazonS3Client == null) {
- AWSCredentials credentials = new BasicAWSCredentials(s3WriterConfiguration.getKey(), s3WriterConfiguration.getSecretKey());
-
- ClientConfiguration clientConfig = new ClientConfiguration();
- clientConfig.setProtocol(Protocol.valueOf(s3WriterConfiguration.getProtocol().toString()));
-
- // We do not want path style access
- S3ClientOptions clientOptions = new S3ClientOptions();
- clientOptions.setPathStyleAccess(false);
-
- this.amazonS3Client = new AmazonS3Client(credentials, clientConfig);
- if (StringUtils.isNotEmpty(s3WriterConfiguration.getRegion())) {
- this.amazonS3Client.setRegion(Region.getRegion(Regions.fromName(s3WriterConfiguration.getRegion())));
- }
- this.amazonS3Client.setS3ClientOptions(clientOptions);
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while preparing the S3 client: {}", ex);
- }
-
- Preconditions.checkArgument(this.amazonS3Client != null);
- }
- }
-
- public void cleanUp() {
- closeAndDestroyWriter();
- }
-
- @Override
- public DatumStatusCounter getDatumStatusCounter() {
- DatumStatusCounter counters = new DatumStatusCounter();
- counters.incrementAttempt(this.totalRecordsWritten.get());
- counters.incrementStatus(DatumStatus.SUCCESS, this.totalRecordsWritten.get());
- return counters;
- }
-}
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json
deleted file mode 100644
index 403bfac..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3Configuration.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.s3.S3Configuration",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {
- "$ref": "../../../../../../../../../../streams-components/streams-converters/src/main/jsonschema/org/apache/streams/converter/LineReadWriteConfiguration.json"
- },
- "properties": {
- "key": {
- "type": "string",
- "description": "Your Amazon Key",
- "required": true
- },
- "secretKey": {
- "type": "string",
- "description": "Your Amazon Secret Key",
- "required": true
- },
- "bucket": {
- "type": "string",
- "description": "Your AWS bucket",
- "required": true
- },
- "protocol": {
- "type": "string",
- "description": "Whether you are using HTTP or HTTPS",
- "enum": ["HTTP", "HTTPS"],
- "default": "HTTPS"
- },
- "region": {
- "type": "string",
- "description": "The AWS region where your bucket resides",
- "required": false
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3ReaderConfiguration.json b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3ReaderConfiguration.json
deleted file mode 100644
index 0b3943f..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3ReaderConfiguration.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.s3.S3ReaderConfiguration",
- "extends": {"$ref":"S3Configuration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "readerPath": {
- "type": "string",
- "description": "Path below root path"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3WriterConfiguration.json b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3WriterConfiguration.json
deleted file mode 100644
index b64a470..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/jsonschema/org/apache/streams/s3/S3WriterConfiguration.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.s3.S3WriterConfiguration",
- "extends": {"$ref":"S3Configuration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "writerPath": {
- "type": "string",
- "description": "Path "
- },
- "writerFilePrefix": {
- "type": "string",
- "description": "File Prefix"
- },
- "maxFileSize": {
- "type": "integer",
- "default" : 20,
- "description": "If files are elected to be 'chunked' which they are by default, this is the maximum size of that file before the byte array stream is vacated and the file is created."
- },
- "chunk": {
- "type": "boolean",
- "default" : true,
- "description": "Whether you want the file chunked inside of a folder or not"
- },
- "compression": {
- "type": "string",
- "description": "compression",
- "enum" : ["none", "gzip"],
- "default": "none"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/resources/components.dot b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/resources/components.dot
deleted file mode 100644
index 0f13c57..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/resources/components.dot
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- processors [label="processors", shape="circle"];
-
- subgraph cluster_persisters {
- label="persisters";
- persisters_s3_reader [label="S3PersistReader"]
- persisters_s3_writer [label="S3PersistWriter"]
- }
-
- subgraph cluster_dbs {
- label="dbs";
- s3 [label="s3"]
- }
-
- generators -> providers
- providers -> processors
- processors -> persisters_s3_writer [label="StreamsDatum"]
- persisters_s3_reader -> processors [label="StreamsDatum[String]"]
- s3 -> persisters_s3_reader
- persisters_s3_writer -> s3
-
-}
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/resources/s3-read.conf b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/resources/s3-read.conf
deleted file mode 100644
index 1d71eb8..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/resources/s3-read.conf
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-s3: {
- key: "",
- secretKey: "",
- bucket: "",
- protocol: "HTTP",
- writerPath: "path/",
- writerFilePrefix: "prefix",
- chunk: false,
- region: "us-east-1"
-}
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/site/markdown/index.md b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/site/markdown/index.md
deleted file mode 100644
index 6f5b365..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/site/markdown/index.md
+++ /dev/null
@@ -1,23 +0,0 @@
-streams-persist-s3
-==============
-
-Write documents to an S3 bucket.
-
-## Configuration
-
-| Schema |
-|--------|
-| [S3Configuration.json](org/apache/streams/s3/S3Configuration.json "S3Configuration.json") [S3Configuration.html](apidocs/org/apache/streams/s3/S3Configuration.html "javadoc") |
-
-## Components
-
-
-
-| Class | Configuration | Example Configuration(s) |
-|-------|---------------|--------------------------|
-| S3PersistReader [S3PersistReader.html](apidocs/org/apache/streams/s3/S3PersistReader.html "javadoc") | [KinesisReaderConfiguration.json](org/apache/streams/s3/KinesisReaderConfiguration.json "KinesisReaderConfiguration.json") [KinesisReaderConfiguration.html](apidocs/org/apache/streams/s3/KinesisReaderConfiguration.html "javadoc") | [s3-read.conf](s3-read.conf "s3-read.conf") |
-| S3PersistWriter [S3PersistWriter.html](apidocs/org/apache/streams/s3/S3PersistWriter "javadoc") | [KinesisWriterConfiguration.json](org/apache/streams/s3/KinesisWriterConfiguration.json "KinesisWriterConfiguration.json") [KinesisWriterConfiguration.html](apidocs/org/apache/streams/s3/KinesisWriterConfiguration.html "javadoc") | [s3-write.conf](s3-write.conf "s3-write.conf") |
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/test/java/org/apache/streams/s3/S3PersistWriterTest.java b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/test/java/org/apache/streams/s3/S3PersistWriterTest.java
deleted file mode 100644
index d585eea..0000000
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/test/java/org/apache/streams/s3/S3PersistWriterTest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.s3;
-
-import org.junit.After;
-import org.junit.Test;
-
-import static org.junit.Assert.assertNotNull;
-
-public class S3PersistWriterTest {
- private S3PersistWriter s3PersistWriter;
-
- @After
- public void tearDown() {
- s3PersistWriter = null;
- }
-
- @Test(expected=IllegalArgumentException.class)
- public void testBadS3Config() {
- s3PersistWriter = new S3PersistWriter(getBadConfig());
-
- s3PersistWriter.prepare(null);
- }
-
- @Test
- public void testGoodS3Config() {
- s3PersistWriter = new S3PersistWriter(getGoodConfig());
-
- s3PersistWriter.prepare(null);
-
- assertNotNull(s3PersistWriter.getAmazonS3Client());
- }
-
- @Test
- public void testCleanup() {
- s3PersistWriter = new S3PersistWriter(getGoodConfig());
-
- s3PersistWriter.prepare(null);
-
- s3PersistWriter.cleanUp();
- }
-
- private S3WriterConfiguration getBadConfig() {
- S3WriterConfiguration s3WriterConfiguration = new S3WriterConfiguration();
-
- s3WriterConfiguration.setWriterPath("bad_path");
- s3WriterConfiguration.setBucket("random_bucket");
-
- return s3WriterConfiguration;
- }
-
- private S3WriterConfiguration getGoodConfig() {
- S3WriterConfiguration s3WriterConfiguration = new S3WriterConfiguration();
-
- s3WriterConfiguration.setWriterPath("good_path/");
- s3WriterConfiguration.setBucket("random_bucket");
- s3WriterConfiguration.setKey("key");
- s3WriterConfiguration.setProtocol(S3Configuration.Protocol.HTTP);
- s3WriterConfiguration.setSecretKey("secret!");
- s3WriterConfiguration.setWriterFilePrefix("prefix");
-
- return s3WriterConfiguration;
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-cassandra/README.md b/streams-contrib/streams-persist-cassandra/README.md
deleted file mode 100644
index cd5ca39..0000000
--- a/streams-contrib/streams-persist-cassandra/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-persist-cassandra
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-cassandra/pom.xml b/streams-contrib/streams-persist-cassandra/pom.xml
deleted file mode 100644
index 13eb817..0000000
--- a/streams-contrib/streams-persist-cassandra/pom.xml
+++ /dev/null
@@ -1,231 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-cassandra</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Cassandra Module</description>
-
- <properties>
- <cassandra.version>3.11.0</cassandra.version>
- <cassandra-driver.version>3.3.0</cassandra-driver.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.cassandra</groupId>
- <artifactId>cassandra-all</artifactId>
- <version>${cassandra.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.datastax.cassandra</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.datastax.cassandra</groupId>
- <artifactId>cassandra-driver-core</artifactId>
- <version>${cassandra-driver.version}</version>
- </dependency>
- <dependency>
- <groupId>com.datastax.cassandra</groupId>
- <artifactId>cassandra-driver-mapping</artifactId>
- <version>${cassandra-driver.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.cassandra.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>resource-dependencies</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <version>${failsafe.plugin.version}</version>
- <configuration>
- <!-- Run integration test suite rather than individual tests. -->
- <excludes>
- <exclude>**/*Test.java</exclude>
- <exclude>**/*Tests.java</exclude>
- </excludes>
- <includes>
- <include>**/*IT.java</include>
- </includes>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <includes>
- <include>**/*.conf</include>
- <include>**/*.json</include>
- <include>**/*.class</include>
- </includes>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <id>dockerITs</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>skipITs</name>
- <value>false</value>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>io.fabric8</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- <configuration combine.self="override">
- <watchInterval>500</watchInterval>
- <logDate>default</logDate>
- <verbose>true</verbose>
- <autoPull>on</autoPull>
- <images>
- <image>
- <name>cassandra:3.9</name>
- <alias>cassandra</alias>
- <run>
- <namingStrategy>none</namingStrategy>
- <ports>
- <port>${cassandra.tcp.host}:${cassandra.tcp.port}:9042</port>
- </ports>
- <portPropertyFile>cassandra.properties</portPropertyFile>
- <log>
- <enabled>true</enabled>
- <date>default</date>
- <color>cyan</color>
- </log>
- </run>
- <watch>
- <mode>none</mode>
- </watch>
- </image>
- </images>
- </configuration>
-
- </plugin>
-
- </plugins>
- </build>
-
- </profile>
- </profiles>
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/CassandraClient.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/CassandraClient.java
deleted file mode 100644
index bbb6b51..0000000
--- a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/CassandraClient.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.Host;
-import com.datastax.driver.core.JdkSSLOptions;
-import com.datastax.driver.core.Metadata;
-import com.datastax.driver.core.SSLOptions;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.SocketOptions;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.net.InetSocketAddress;
-import java.security.KeyStore;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Objects;
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManagerFactory;
-
-import static com.datastax.driver.core.SocketOptions.DEFAULT_CONNECT_TIMEOUT_MILLIS;
-import static com.datastax.driver.core.SocketOptions.DEFAULT_READ_TIMEOUT_MILLIS;
-
-public class CassandraClient {
-
- private static final Logger LOGGER = LoggerFactory
- .getLogger(CassandraClient.class);
-
- private Cluster cluster;
- private Session session;
-
- public CassandraConfiguration config;
-
- public CassandraClient(CassandraConfiguration config) throws Exception {
- this.config = config;
- org.apache.cassandra.config.Config.setClientMode(true);
- }
-
- public void start() throws Exception {
-
- Objects.nonNull(config);
-
- LOGGER.info("CassandraClient.start {}", config);
-
- Cluster.Builder builder = Cluster.builder()
- .withPort(config.getPort().intValue())
- .withoutJMXReporting()
- .withoutMetrics()
- .withSocketOptions(
- new SocketOptions()
- .setConnectTimeoutMillis(DEFAULT_CONNECT_TIMEOUT_MILLIS*10)
- .setReadTimeoutMillis(DEFAULT_READ_TIMEOUT_MILLIS*10)
- );
-
- if( config.getSsl() != null && config.getSsl().getEnabled() == true) {
-
- Ssl ssl = config.getSsl();
-
- KeyStore ks = KeyStore.getInstance("JKS");
-
- InputStream trustStore = new FileInputStream(ssl.getTrustStore());
- ks.load(trustStore, ssl.getTrustStorePassword().toCharArray());
- InputStream keyStore = new FileInputStream(ssl.getKeyStore());
- ks.load(keyStore, ssl.getKeyStorePassword().toCharArray());
-
- TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
- tmf.init(ks);
-
- KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
- kmf.init(ks, ssl.getKeyStorePassword().toCharArray());
-
- SSLContext sslContext = SSLContext.getInstance("SSLv3");
- sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
-
- SSLOptions sslOptions = JdkSSLOptions.builder()
- .withSSLContext(sslContext)
- .build();
-
- builder = builder.withSSL(sslOptions);
- }
-
- Collection<InetSocketAddress> addresses = new ArrayList<>();
- for (String h : config.getHosts()) {
- LOGGER.info("Adding Host: {}", h);
- InetSocketAddress socketAddress = new InetSocketAddress(h, config.getPort().intValue());
- addresses.add(socketAddress);
- }
- builder.addContactPointsWithPorts(addresses);
-
- if( StringUtils.isNotBlank(config.getUser()) &&
- StringUtils.isNotBlank(config.getPassword())) {
- builder.withCredentials(config.getUser(), config.getPassword());
- }
- cluster = builder.build();
-
- Objects.nonNull(cluster);
-
- try {
- Metadata metadata = cluster.getMetadata();
- LOGGER.info("Connected to cluster: {}\n",
- metadata.getClusterName());
- for ( Host host : metadata.getAllHosts() ) {
- LOGGER.info("Datacenter: {}; Host: {}; Rack: {}\n",
- host.getDatacenter(), host.getAddress(), host.getRack());
- }
- } catch( Exception e ) {
- LOGGER.error("Exception: {}", e);
- throw e;
- }
-
- try {
- session = cluster.connect();
- } catch( Exception e ) {
- LOGGER.error("Exception: {}", e);
- throw e;
- }
-
- Objects.nonNull(session);
-
- }
-
- public void stop() throws Exception {
- session.close();
- cluster.close();
- }
-
- public CassandraConfiguration config() {
- return config;
- }
-
- public Session client() {
- return session;
- }
-
- public Cluster cluster() {
- return cluster;
- }
-}
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/CassandraPersistReader.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/CassandraPersistReader.java
deleted file mode 100644
index 251a02d..0000000
--- a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/CassandraPersistReader.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.DataType;
-import com.datastax.driver.core.KeyspaceMetadata;
-import com.datastax.driver.core.Metadata;
-import com.datastax.driver.core.ResultSet;
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.TableMetadata;
-import com.datastax.driver.core.querybuilder.QueryBuilder;
-import com.datastax.driver.core.schemabuilder.SchemaBuilder;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Queues;
-import org.apache.commons.lang3.StringUtils;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.math.BigInteger;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * CassandraPersistReader reads documents from cassandra.
- */
-public class CassandraPersistReader implements StreamsPersistReader {
-
- public static final String STREAMS_ID = "CassandraPersistReader";
-
- public static final Logger LOGGER = LoggerFactory.getLogger(CassandraPersistReader.class);
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private ExecutorService executor;
- private CompletableFuture<Boolean> readerTaskFuture = new CompletableFuture<>();
-
- private CassandraConfiguration config;
- private CassandraClient client;
-
- protected Iterator<Row> rowIterator;
-
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- /**
- * CassandraPersistReader constructor - resolves CassandraConfiguration from JVM 'cassandra'.
- */
- public CassandraPersistReader() {
- this.config = new ComponentConfigurator<>(CassandraConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("cassandra"));
- }
-
- /**
- * CassandraPersistReader constructor - uses supplied CassandraConfiguration.
- * @param config config
- */
- public CassandraPersistReader(CassandraConfiguration config) {
- this.config = config;
- }
-
- /**
- * CassandraPersistReader constructor - uses supplied persistQueue.
- * @param persistQueue persistQueue
- */
- public CassandraPersistReader(Queue<StreamsDatum> persistQueue) {
- this.config = new ComponentConfigurator<>(CassandraConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("cassandra"));
- this.persistQueue = persistQueue;
- }
-
- public void setPersistQueue(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
- }
-
- public Queue<StreamsDatum> getPersistQueue() {
- return persistQueue;
- }
-
- public void stop() {
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- try {
- connectToCassandra();
- client.start();
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- return;
- }
-
- String selectStatement = getSelectStatement();
- ResultSet rs = client.client().execute(selectStatement);
- rowIterator = rs.iterator();
-
- if (!rowIterator.hasNext()) {
- throw new RuntimeException("Table" + config.getTable() + "is empty!");
- }
-
- persistQueue = constructQueue();
-
- executor = Executors.newSingleThreadExecutor();
- }
-
- @Override
- public void cleanUp() {
- stop();
- }
-
- protected StreamsDatum prepareDatum(Row row) {
- ObjectNode objectNode;
-
- try {
- byte[] value = row.getBytes(config.getColumn()).array();
- objectNode = mapper.readValue(value, ObjectNode.class);
- } catch (IOException ex) {
- LOGGER.warn("document isn't valid JSON.");
- return null;
- }
-
- return new StreamsDatum(objectNode);
- }
-
- private synchronized void connectToCassandra() throws Exception {
-
- client = new CassandraClient(config);
-
- }
-
- @Override
- public StreamsResultSet readAll() {
- ResultSet rs = client.client().execute(getSelectStatement());
- Iterator<Row> rowsIterator = rs.iterator();
-
- while (rowsIterator.hasNext()) {
- Row row = rowsIterator.next();
- StreamsDatum datum = prepareDatum(row);
- write(datum);
- }
-
- return readCurrent();
- }
-
- @Override
- public void startStream() {
- LOGGER.debug("startStream");
- CassandraPersistReaderTask readerTask = new CassandraPersistReaderTask(this);
-
- CompletableFuture.runAsync(readerTask, executor);
-
- try {
- if (readerTaskFuture.get()) {
- executor.shutdown();
- }
- } catch (InterruptedException ex) {
- LOGGER.trace("Interrupt", ex);
- } catch (ExecutionException ex) {
- LOGGER.trace("Execution exception", ex);
- }
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- try {
- lock.writeLock().lock();
- current = new StreamsResultSet(persistQueue);
- current.setCounter(new DatumStatusCounter());
- persistQueue = constructQueue();
- } finally {
- lock.writeLock().unlock();
- }
-
- return current;
- }
-
- protected void write(StreamsDatum entry) {
- boolean success;
- do {
- try {
- lock.readLock().lock();
- success = persistQueue.offer(entry);
- Thread.yield();
- } finally {
- lock.readLock().unlock();
- }
- }
- while (!success);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return !executor.isTerminated() || !executor.isShutdown();
- }
-
- private Queue<StreamsDatum> constructQueue() {
- return Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
- }
-
- private String getSelectStatement() {
- return QueryBuilder.select().all()
- .from(config.getKeyspace(), config.getTable())
- .getQueryString();
- }
-
- public class CassandraPersistReaderTask implements Runnable {
-
- private CassandraPersistReader reader;
-
- public CassandraPersistReaderTask(CassandraPersistReader reader) {
- this.reader = reader;
- }
-
- @Override
- public void run() {
- try {
- while (reader.rowIterator.hasNext()) {
- Row row = reader.rowIterator.next();
- StreamsDatum datum = reader.prepareDatum(row);
- reader.write(datum);
- }
- } finally {
- readerTaskFuture.complete(true);
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/CassandraPersistWriter.java b/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/CassandraPersistWriter.java
deleted file mode 100644
index 9f419bd..0000000
--- a/streams-contrib/streams-persist-cassandra/src/main/java/org/apache/streams/cassandra/CassandraPersistWriter.java
+++ /dev/null
@@ -1,316 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.GuidUtils;
-
-import com.datastax.driver.core.BatchStatement;
-import com.datastax.driver.core.BoundStatement;
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.DataType;
-import com.datastax.driver.core.KeyspaceMetadata;
-import com.datastax.driver.core.Metadata;
-import com.datastax.driver.core.PreparedStatement;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.TableMetadata;
-import com.datastax.driver.core.querybuilder.Insert;
-import com.datastax.driver.core.querybuilder.QueryBuilder;
-import com.datastax.driver.core.schemabuilder.SchemaBuilder;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.Flushable;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.Random;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-public class CassandraPersistWriter implements StreamsPersistWriter, Runnable, Flushable, Closeable {
-
- public static final String STREAMS_ID = "CassandraPersistWriter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(CassandraPersistWriter.class);
-
- private static final long MAX_WRITE_LATENCY = 1000;
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- private volatile AtomicLong lastWrite = new AtomicLong(System.currentTimeMillis());
- private ScheduledExecutorService backgroundFlushTask = Executors.newSingleThreadScheduledExecutor();
-
- private CassandraConfiguration config;
- private CassandraClient client;
-
- private Session session;
-
- protected PreparedStatement insertStatement;
-
- protected List<BoundStatement> insertBatch = new ArrayList<>();
-
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- public CassandraPersistWriter() {
- this(new ComponentConfigurator<>(CassandraConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("cassandra")));
- }
-
- public CassandraPersistWriter(CassandraConfiguration config) {
- this.config = config;
- }
-
- public void setPersistQueue(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
- }
-
- public Queue<StreamsDatum> getPersistQueue() {
- return persistQueue;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void write(StreamsDatum streamsDatum) {
-
- ObjectNode node;
-
- if (streamsDatum.getDocument() instanceof String) {
- try {
- node = mapper.readValue((String) streamsDatum.getDocument(), ObjectNode.class);
-
- byte[] value = node.toString().getBytes();
-
- String key = GuidUtils.generateGuid(node.toString());
- if(!Objects.isNull(streamsDatum.getMetadata().get("id"))) {
- key = streamsDatum.getMetadata().get("id").toString();
- }
-
- BoundStatement statement = insertStatement.bind(key, ByteBuffer.wrap(value));
- insertBatch.add(statement);
- } catch (IOException ex) {
- LOGGER.warn("Failure adding object: {}", streamsDatum.getDocument().toString());
- return;
- }
- } else {
- try {
- node = mapper.valueToTree(streamsDatum.getDocument());
-
- byte[] value = node.toString().getBytes();
-
- String key = GuidUtils.generateGuid(node.toString());
- if(!Objects.isNull(streamsDatum.getId())) {
- key = streamsDatum.getId();
- }
-
- BoundStatement statement = insertStatement.bind(key, ByteBuffer.wrap(value));
- insertBatch.add(statement);
- } catch (Exception ex) {
- LOGGER.warn("Failure adding object: {}", streamsDatum.getDocument().toString());
- return;
- }
- }
-
- flushIfNecessary();
- }
-
- @Override
- public void flush() throws IOException {
- try {
- LOGGER.debug("Attempting to flush {} items to cassandra", insertBatch.size());
- lock.writeLock().lock();
-
- BatchStatement batchStatement = new BatchStatement();
- batchStatement.addAll(insertBatch);
- session.execute(batchStatement);
-
- lastWrite.set(System.currentTimeMillis());
- insertBatch = new ArrayList<>();
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- @Override
- public synchronized void close() throws IOException {
- session.close();
- client.cluster().close();
- backgroundFlushTask.shutdownNow();
- }
-
- /**
- * start write thread.
- */
- public void start() {
- try {
- connectToCassandra();
- client.start();
- createKeyspaceAndTable();
- createInsertStatement();
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- return;
- }
- backgroundFlushTask.scheduleAtFixedRate(new Runnable() {
- @Override
- public void run() {
- flushIfNecessary();
- }
- }, 0, MAX_WRITE_LATENCY * 2, TimeUnit.MILLISECONDS);
- }
-
- /**
- * stop.
- */
- public void stop() {
- try {
- flush();
- } catch (IOException ex) {
- LOGGER.error("Error flushing", ex);
- }
-
- try {
- close();
- } catch (IOException ex) {
- LOGGER.error("Error closing", ex);
- }
-
- try {
- backgroundFlushTask.shutdown();
- // Wait a while for existing tasks to terminate
- if (!backgroundFlushTask.awaitTermination(15, TimeUnit.SECONDS)) {
- backgroundFlushTask.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!backgroundFlushTask.awaitTermination(15, TimeUnit.SECONDS)) {
- LOGGER.error("Stream did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- backgroundFlushTask.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
- @Override
- public void run() {
- while (true) {
- if (persistQueue.peek() != null) {
- try {
- StreamsDatum entry = persistQueue.remove();
- write(entry);
- } catch (Exception ex) {
- LOGGER.warn("Failure writing entry from Queue: {}", ex.getMessage());
- }
- }
- try {
- Thread.sleep(new Random().nextInt(1));
- } catch (InterruptedException interrupt) {
- LOGGER.trace("Interrupt", interrupt);
- }
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.persistQueue = new ConcurrentLinkedQueue<>();
- start();
- }
-
- @Override
- public void cleanUp() {
- stop();
- }
-
- protected void flushIfNecessary() {
- long lastLatency = System.currentTimeMillis() - lastWrite.get();
- //Flush iff the size > 0 AND the size is divisible by 100 or the time between now and the last flush is greater
- //than the maximum desired latency
- if (insertBatch.size() > 0 && (insertBatch.size() % 100 == 0 || lastLatency > MAX_WRITE_LATENCY)) {
- try {
- flush();
- } catch (IOException ex) {
- LOGGER.error("Error writing to Cassandra", ex);
- }
- }
- }
-
- private synchronized void connectToCassandra() throws Exception {
- client = new CassandraClient(config);
- }
-
- private void createKeyspaceAndTable() {
- Metadata metadata = client.cluster().getMetadata();
- if (Objects.isNull(metadata.getKeyspace(config.getKeyspace()))) {
- LOGGER.info("Keyspace {} does not exist. Creating Keyspace", config.getKeyspace());
- Map<String, Object> replication = new HashMap<>();
- replication.put("class", "SimpleStrategy");
- replication.put("replication_factor", 1);
-
- String createKeyspaceStmt = SchemaBuilder.createKeyspace(config.getKeyspace()).with()
- .replication(replication).getQueryString();
- client.cluster().connect().execute(createKeyspaceStmt);
- }
-
- session = client.cluster().connect(config.getKeyspace());
-
- KeyspaceMetadata ks = metadata.getKeyspace(config.getKeyspace());
- TableMetadata tableMetadata = ks.getTable(config.getTable());
-
- if (Objects.isNull(tableMetadata)) {
- LOGGER.info("Table {} does not exist in Keyspace {}. Creating Table", config.getTable(), config.getKeyspace());
- String createTableStmt = SchemaBuilder.createTable(config.getTable())
- .addPartitionKey(config.getPartitionKeyColumn(), DataType.varchar())
- .addColumn(config.getColumn(), DataType.blob()).getQueryString();
-
- session.execute(createTableStmt);
- }
- }
-
- private void createInsertStatement() {
- Insert insertBuilder = QueryBuilder.insertInto(config.getTable());
- insertBuilder.value(config.getPartitionKeyColumn(), new Object());
- insertBuilder.value(config.getColumn(), new Object());
- insertStatement = session.prepare(insertBuilder.getQueryString());
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-cassandra/src/main/jsonschema/org/apache/streams/cassandra/CassandraConfiguration.json b/streams-contrib/streams-persist-cassandra/src/main/jsonschema/org/apache/streams/cassandra/CassandraConfiguration.json
deleted file mode 100644
index ea522a4..0000000
--- a/streams-contrib/streams-persist-cassandra/src/main/jsonschema/org/apache/streams/cassandra/CassandraConfiguration.json
+++ /dev/null
@@ -1,70 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.cassandra.CassandraConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "hosts": {
- "type": "array",
- "items": {
- "type": "string"
- },
- "description": "Cassandra hosts"
- },
- "port": {
- "type": "integer",
- "description": "Cassandra port"
- },
- "user": {
- "type": "string",
- "description": "User"
- },
- "password": {
- "type": "string",
- "description": "Password"
- },
- "keyspace": {
- "type": "string",
- "description": "Keyspace"
- },
- "table": {
- "type": "string",
- "description": "Table"
- },
- "partitionKeyColumn": {
- "type": "string",
- "description": "Partition Key column name"
- },
- "column": {
- "type": "string",
- "description": "Column name"
- },
- "ssl": {
- "type": "object",
- "description": "ssl details",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "enabled": {
- "type": "boolean",
- "description": "ssl enabled"
- },
- "trustStore": {
- "type": "string"
- },
- "trustStorePassword": {
- "type": "string"
- },
- "keyStore": {
- "type": "string"
- },
- "keyStorePassword": {
- "type": "string"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-cassandra/src/main/resources/components.dot b/streams-contrib/streams-persist-cassandra/src/main/resources/components.dot
deleted file mode 100644
index 916d8c0..0000000
--- a/streams-contrib/streams-persist-cassandra/src/main/resources/components.dot
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- processors [label="processors", shape="circle"];
-
- subgraph cluster_persisters {
- label="persisters";
- persisters_cassandra_reader [label="CassandraPersistReader"]
- persisters_cassandra_writer [label="CassandraPersistWriter"]
- }
-
- subgraph cluster_dbs {
- label="dbs";
- cassandra [label="cassandra", shape="cylinder"]
- }
-
- generators -> providers
- providers -> processors
- processors -> persisters_cassandra_writer [label="StreamsDatum"]
- persisters_cassandra_reader -> processors [label="StreamsDatum[String]"]
- cassandra -> persisters_cassandra_reader
- persisters_cassandra_writer -> cassandra
-
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-cassandra/src/main/resources/reference.conf b/streams-contrib/streams-persist-cassandra/src/main/resources/reference.conf
deleted file mode 100644
index 6a18dc6..0000000
--- a/streams-contrib/streams-persist-cassandra/src/main/resources/reference.conf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-cassandra {
- host = ["127.0.0.1"]
- port = 9042
- keyspace = "test_keyspace"
- table = "test_table"
- partitionKeyColumn = "key"
- column = value
- ssl {
- enabled = false
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-cassandra/src/site/markdown/cassandra.md b/streams-contrib/streams-persist-cassandra/src/site/markdown/cassandra.md
deleted file mode 100644
index 5799bdb..0000000
--- a/streams-contrib/streams-persist-cassandra/src/site/markdown/cassandra.md
+++ /dev/null
@@ -1,36 +0,0 @@
-## Cassandra
-
-Start cassandra via docker with the docker maven plugin:
-
- mvn -PdockerITs docker:start
-
-Confirm that cassandra is running:
-
- docker ps
-
-Confirm that host and post(s) are in property file:
-
- cat cassandra.properties
-
-Create a local file `cassandra.conf` with cluster details:
-
- cassandra {
- host = ${cassandra.tcp.host}
- port = ${cassandra.tcp.port}
- }
-
-When configuring a stream, include these files:
-
- include "cassandra.properties"
- include "cassandra.conf"
-
-Supply application-specific configuration as well:
-
- cassandra {
- keyspace = test_keyspace1
- table = test_table1
- partitionKeyColumn = key
- column = value
- }
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-cassandra/src/site/markdown/index.md b/streams-contrib/streams-persist-cassandra/src/site/markdown/index.md
deleted file mode 100644
index 353f1c4..0000000
--- a/streams-contrib/streams-persist-cassandra/src/site/markdown/index.md
+++ /dev/null
@@ -1,23 +0,0 @@
-streams-persist-cassandra
-=====================
-
-Read/write to/from Cassandra
-
-## Configuration
-
-| Schema |
-|--------|
-| [CassandraConfiguration.json](../../../org/apache/streams/cassandra/CassandraConfiguration.json "CassandraConfiguration.json") [CassandraConfiguration.html](apidocs/org/apache/streams/cassandra/CassandraConfiguration.html "javadoc") |
-
-## Components
-
-
-
-| Class |
-|-------|
-| CassandraPersistReader [CassandraPersistReader.html](apidocs/org/apache/streams/cassandra/CassandraPersistReader.html "javadoc")
-| CassandraPersistWriter [CassandraPersistWriter.html](apidocs/org/apache/streams/cassandra/CassandraPersistWriter.html "javadoc")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-cassandra/src/site/site.xml b/streams-contrib/streams-persist-cassandra/src/site/site.xml
deleted file mode 100644
index f82300a..0000000
--- a/streams-contrib/streams-persist-cassandra/src/site/site.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project>
- <body>
- <links name="Help">
- <item name="Cassandra" href="cassandra.html"/>
- </links>
- </body>
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/test/CassandraPersistIT.java b/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/test/CassandraPersistIT.java
deleted file mode 100644
index aa2f16d..0000000
--- a/streams-contrib/streams-persist-cassandra/src/test/java/org/apache/streams/cassandra/test/CassandraPersistIT.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.cassandra.test;
-
-import org.apache.streams.cassandra.CassandraConfiguration;
-import org.apache.streams.cassandra.CassandraPersistReader;
-import org.apache.streams.cassandra.CassandraPersistWriter;
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-/**
- * Test writing documents
- */
-public class CassandraPersistIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(CassandraPersistIT.class);
-
- private ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private CassandraConfiguration testConfiguration;
-
- private int count = 0;
-
- @BeforeClass
- public void setup() throws Exception {
- Config reference = ConfigFactory.load();
- File conf_file = new File("target/test-classes/CassandraPersistIT.conf");
- assert(conf_file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(CassandraConfiguration.class).detectConfiguration(typesafe, "cassandra");
- }
-
- @Test
- public void testCassandraPersist() throws Exception {
- CassandraPersistWriter writer = new CassandraPersistWriter(testConfiguration);
-
- writer.prepare(null);
-
- InputStream testActivityFolderStream = CassandraPersistIT.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- for (String file: files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = CassandraPersistIT.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- activity.getAdditionalProperties().remove("$license");
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- writer.write(datum);
-
- LOGGER.info("Wrote: " + activity.getVerb() );
- count++;
- }
-
- LOGGER.info("Total Written: {}", count );
- Assert.assertEquals(89, count);
-
- writer.cleanUp();
-
- CassandraPersistReader reader = new CassandraPersistReader(testConfiguration);
-
- reader.prepare(null);
-
- StreamsResultSet resultSet = reader.readAll();
-
- LOGGER.info("Total Read: {}", resultSet.size() );
- Assert.assertEquals(resultSet.size(), 89);
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-cassandra/src/test/resources/CassandraPersistIT.conf b/streams-contrib/streams-persist-cassandra/src/test/resources/CassandraPersistIT.conf
deleted file mode 100644
index 9db9da5..0000000
--- a/streams-contrib/streams-persist-cassandra/src/test/resources/CassandraPersistIT.conf
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-cassandra {
- hosts = [${cassandra.tcp.host}]
- port = ${cassandra.tcp.port}
- user = cassandra
- password = cassandra
- keyspace = test_keyspace
- table = test_table
- partitionKeyColumn = key
- column = value
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-cassandra/src/test/resources/cassandra.conf b/streams-contrib/streams-persist-cassandra/src/test/resources/cassandra.conf
deleted file mode 100644
index 8576e9a..0000000
--- a/streams-contrib/streams-persist-cassandra/src/test/resources/cassandra.conf
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-cassandra {
- "host": ["127.0.0.1"],
- "port": 9042,
- "keyspace": "test_keyspace",
- "table": "test_table",
- "partitionKeyColumn": "key",
- "column": "value"
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-console/README.md b/streams-contrib/streams-persist-console/README.md
deleted file mode 100644
index 237c025..0000000
--- a/streams-contrib/streams-persist-console/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-streams-persist-console
-=====================
-
-Read and write documents from console
-
-This can also be used to read and write files.
-
-For example, to read a stream from a file of json documents:
-
- cat inputfile.txt | java -jar stream.jar
-
-For example, to write a stream to a file of json documents:
-
- java -jar stream.jar > outputfile.txt
-
-NOTE: Be sure to disable the console logger in the latter case.
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-console/pom.xml b/streams-contrib/streams-persist-console/pom.xml
deleted file mode 100644
index 8ca55d8..0000000
--- a/streams-contrib/streams-persist-console/pom.xml
+++ /dev/null
@@ -1,85 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-console</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Console Module</description>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-runtime-local</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- </build>
-
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
deleted file mode 100644
index 43d9e34..0000000
--- a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistReader.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.console;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.InputStream;
-import java.math.BigInteger;
-import java.util.Queue;
-import java.util.Scanner;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-/**
- * ConsolePersistReader reads documents from stdin.
- */
-public class ConsolePersistReader implements StreamsPersistReader {
-
- private static final String STREAMS_ID = "ConsolePersistReader";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ConsolePersistReader.class);
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- protected InputStream inputStream = System.in;
-
- public ConsolePersistReader() {
- this.persistQueue = new ConcurrentLinkedQueue<StreamsDatum>();
- }
-
- public ConsolePersistReader(InputStream inputStream) {
- this();
- this.inputStream = inputStream;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- public void prepare(Object configuration) {
-
- }
-
- public void cleanUp() {
-
- }
-
- @Override
- public void startStream() {
- // no op
- }
-
- @Override
- public StreamsResultSet readAll() {
- return readCurrent();
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- LOGGER.info("{} readCurrent", STREAMS_ID);
-
- Scanner sc = new Scanner(inputStream);
-
- while ( sc.hasNextLine() ) {
-
- persistQueue.offer(new StreamsDatum(sc.nextLine()));
-
- }
-
- LOGGER.info("Providing {} docs", persistQueue.size());
-
- StreamsResultSet result = new StreamsResultSet(persistQueue);
-
- LOGGER.info("{} Exiting", STREAMS_ID);
-
- return result;
-
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return readCurrent();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return readCurrent();
- }
-
- @Override
- public boolean isRunning() {
- return true; //Will always be running
- }
-}
diff --git a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
deleted file mode 100644
index dfbd91e..0000000
--- a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriter.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.console;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.PrintStream;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-/**
- * ConsolePersistWriter writes documents to stdout.
- */
-public class ConsolePersistWriter implements StreamsPersistWriter {
-
- private static final String STREAMS_ID = "ConsolePersistWriter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ConsolePersistWriter.class);
-
- private PrintStream printStream = System.out;
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- public ConsolePersistWriter() {
- this.persistQueue = new ConcurrentLinkedQueue<>();
- }
-
- public ConsolePersistWriter(PrintStream printStream) {
- this();
- this.printStream = printStream;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- public void prepare(Object configuration) {
- Objects.requireNonNull(persistQueue);
- }
-
- public void cleanUp() {
-
- }
-
- @Override
- public void write(StreamsDatum entry) {
-
- try {
-
- String text = mapper.writeValueAsString(entry);
-
- printStream.println(text);
-
- } catch (JsonProcessingException ex) {
- LOGGER.warn("save: {}", ex);
- }
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriterTask.java b/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriterTask.java
deleted file mode 100644
index ecb60e3..0000000
--- a/streams-contrib/streams-persist-console/src/main/java/org/apache/streams/console/ConsolePersistWriterTask.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.console;
-
-import org.apache.streams.core.StreamsDatum;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Random;
-
-/**
- * ConsolePersistWriterTask writes documents to stdout on behalf of
- * @see org.apache.streams.console.ConsolePersistWriter
- */
-public class ConsolePersistWriterTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ConsolePersistWriterTask.class);
-
- private ConsolePersistWriter writer;
-
- public ConsolePersistWriterTask(ConsolePersistWriter writer) {
- this.writer = writer;
- }
-
- @Override
- public void run() {
- while (true) {
- if ( writer.persistQueue.peek() != null ) {
- try {
- StreamsDatum entry = writer.persistQueue.remove();
- writer.write(entry);
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
- try {
- Thread.sleep(new Random().nextInt(100));
- } catch (InterruptedException interrupt) {
- LOGGER.trace("Interrupted", interrupt);
- }
- }
- }
-
-}
diff --git a/streams-contrib/streams-persist-console/src/main/resources/components.dot b/streams-contrib/streams-persist-console/src/main/resources/components.dot
deleted file mode 100644
index 1f29e1c..0000000
--- a/streams-contrib/streams-persist-console/src/main/resources/components.dot
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- processors [label="processors", shape="circle"];
-
- subgraph cluster_persisters {
- label="persisters";
- persisters_console_reader [label="KinesisPersistReader"]
- persisters_console_writer [label="KinesisPersistWriter"]
- }
-
- generators -> providers
- providers -> processors
- processors -> persisters_console_writer [label="StreamsDatum"]
- persisters_console_reader -> processors [label="StreamsDatum[String]"]
-
-}
diff --git a/streams-contrib/streams-persist-console/src/site/markdown/index.md b/streams-contrib/streams-persist-console/src/site/markdown/index.md
deleted file mode 100644
index 19f1684..0000000
--- a/streams-contrib/streams-persist-console/src/site/markdown/index.md
+++ /dev/null
@@ -1,17 +0,0 @@
-streams-persist-console
-=======================
-
-Read/Write documents to/from console.
-
-## Components
-
-
-
-| Class |
-|-------|
-| ConsolePersistReader [ConsolePersistReader.html](apidocs/org/apache/streams/console/ConsolePersistReader.html "javadoc")
-| ConsolePersistWriter [ConsolePersistWriter.html](apidocs/org/apache/streams/console/ConsolePersistWriter "javadoc")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-elasticsearch/README.md b/streams-contrib/streams-persist-elasticsearch/README.md
deleted file mode 100644
index 5180dea..0000000
--- a/streams-contrib/streams-persist-elasticsearch/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-elasticsearch
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-persist-elasticsearch/pom.xml b/streams-contrib/streams-persist-elasticsearch/pom.xml
deleted file mode 100644
index d228749..0000000
--- a/streams-contrib/streams-persist-elasticsearch/pom.xml
+++ /dev/null
@@ -1,294 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-elasticsearch</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Elasticsearch Module</description>
-
- <properties>
- <elasticsearch.version>2.4.6</elasticsearch.version>
- <lucene.version>5.5.4</lucene.version>
- </properties>
-
- <dependencies>
- <!-- Test includes -->
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-test-framework</artifactId>
- <version>${lucene.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-codecs</artifactId>
- <version>${lucene.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-all</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <scope>compile</scope>
- <type>jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>${elasticsearch.version}</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <type>test-jar</type>
- <version>${elasticsearch.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-test-framework</artifactId>
- <version>${lucene.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-codecs</artifactId>
- <version>${lucene.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.elasticsearch.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>test-jar</includeTypes>
- </configuration>
- <executions>
- <execution>
- <id>test-resource-dependencies</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <version>${failsafe.plugin.version}</version>
- <configuration>
- <!-- Run integration test suite rather than individual tests. -->
- <excludes>
- <exclude>**/*Test.java</exclude>
- <exclude>**/*Tests.java</exclude>
- </excludes>
- <includes>
- <include>**/*IT.java</include>
- </includes>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <includes>
- <include>**/*.conf</include>
- <include>**/*.json</include>
- <include>**/*.class</include>
- </includes>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <id>dockerITs</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>skipITs</name>
- <value>false</value>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>io.fabric8</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- <configuration combine.self="override">
- <watchInterval>500</watchInterval>
- <logDate>default</logDate>
- <verbose>true</verbose>
- <autoPull>on</autoPull>
- <images>
- <image>
- <name>elasticsearch:2.3.5</name>
- <alias>elasticsearch</alias>
- <run>
- <namingStrategy>none</namingStrategy>
- <ports>
- <port>${es.http.host}:${es.http.port}:9200</port>
- <port>${es.tcp.host}:${es.tcp.port}:9300</port>
- </ports>
- <portPropertyFile>elasticsearch.properties</portPropertyFile>
- <wait>
- <log>elasticsearch startup</log>
- <http>
- <url>http://${es.http.host}:${es.http.port}</url>
- <method>GET</method>
- <status>200</status>
- </http>
- <time>20000</time>
- <kill>1000</kill>
- <shutdown>500</shutdown>
- <!--<tcp>-->
- <!--<host>${es.transport.host}</host>-->
- <!--<ports>-->
- <!--<port>${es.transport.port}</port>-->
- <!--</ports>-->
- <!--</tcp>-->
- </wait>
- <log>
- <enabled>true</enabled>
- <date>default</date>
- <color>cyan</color>
- </log>
- </run>
- <watch>
- <mode>none</mode>
- </watch>
- </image>
-
- </images>
- </configuration>
-
- </plugin>
-
- </plugins>
- </build>
-
- </profile>
- </profiles>
-
-</project>
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java
deleted file mode 100644
index ceb173a..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch;
-
-import com.google.common.net.InetAddresses;
-
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.InetAddress;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Wrapper class for multiple
- * @see org.apache.streams.elasticsearch.ElasticsearchClient
- */
-public class ElasticsearchClientManager {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchClientManager.class);
-
- private ElasticsearchConfiguration config;
-
- private org.elasticsearch.client.transport.TransportClient client;
-
- private ElasticsearchClientManager(ElasticsearchConfiguration config) {
- this.config = config;
- try {
- this.start();
- } catch (Exception e) {
- e.printStackTrace();
- this.client = null;
- }
- }
-
- private static Map<ElasticsearchConfiguration, ElasticsearchClientManager> INSTANCE_MAP = new HashMap<>();
-
- public static ElasticsearchClientManager getInstance(ElasticsearchConfiguration configuration) {
- if (INSTANCE_MAP != null &&
- INSTANCE_MAP.size() > 0 &&
- INSTANCE_MAP.containsKey(configuration)
- ) {
- return INSTANCE_MAP.get(configuration);
- } else {
- ElasticsearchClientManager instance = new ElasticsearchClientManager(configuration);
- if (instance != null) {
- INSTANCE_MAP.put(configuration, instance);
- return instance;
- } else {
- return null;
- }
- }
- }
-
- private synchronized void start() {
-
- try {
- // We are currently using lazy loading to start the elasticsearch cluster, however.
- LOGGER.info("Creating a new TransportClient: {}", this.config.getHosts());
-
- Settings settings = Settings.settingsBuilder()
- .put("cluster.name", this.config.getClusterName())
- .put("client.transport.ping_timeout", "90s")
- .put("client.transport.nodes_sampler_interval", "60s")
- .build();
-
- // Create the client
- client = TransportClient.builder().settings(settings).build();
- for (String h : config.getHosts()) {
- LOGGER.info("Adding Host: {}", h);
- InetAddress address;
-
- if ( InetAddresses.isInetAddress(h)) {
- LOGGER.info("{} is an IP address", h);
- address = InetAddresses.forString(h);
- } else {
- LOGGER.info("{} is a hostname", h);
- address = InetAddress.getByName(h);
- }
- client.addTransportAddress(
- new InetSocketTransportAddress(
- address,
- config.getPort().intValue()));
- }
- } catch (Exception ex) {
- LOGGER.error("Could not Create elasticsearch Transport Client: {}", ex);
- }
-
- }
-
- public ElasticsearchConfiguration config() {
- return config;
- }
-
- public org.elasticsearch.client.transport.TransportClient client() {
- return client;
- }
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchMetadataUtil.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchMetadataUtil.java
deleted file mode 100644
index 8fbbf3c..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchMetadataUtil.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch;
-
-import org.apache.streams.core.StreamsDatum;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * Utility class for handling Elasticsearch Metadata maps.
- */
-public class ElasticsearchMetadataUtil {
-
- /**
- * get Index to use based on supplied parameters.
- *
- * @param metadata metadata
- * @param config config
- * @return result
- */
- public static String getIndex(Map<String, Object> metadata, ElasticsearchWriterConfiguration config) {
-
- String index = null;
-
- if ( metadata != null && metadata.containsKey("index")) {
- index = (String) metadata.get("index");
- }
-
- if ( index == null || (config.getForceUseConfig() != null && config.getForceUseConfig())) {
- index = config.getIndex();
- }
-
- return index;
- }
-
- /**
- * get Index to use based on supplied parameters.
- *
- * @param metadata metadata
- * @param config config
- * @return result
- */
- public static String getIndex(Map<String, Object> metadata, ElasticsearchReaderConfiguration config) {
-
- String index = null;
-
- if ( metadata != null && metadata.containsKey("index")) {
- index = (String) metadata.get("index");
- }
-
- if ( index == null ) {
- index = config.getIndexes().get(0);
- }
-
- return index;
- }
-
- /**
- * get Type to use based on supplied parameters.
- *
- * @param metadata metadata
- * @param config config
- * @return result
- */
- public static String getType(Map<String, Object> metadata, ElasticsearchWriterConfiguration config) {
-
- String type = null;
-
- if ( metadata != null && metadata.containsKey("type")) {
- type = (String) metadata.get("type");
- }
-
- if (type == null || (config.getForceUseConfig() != null && config.getForceUseConfig())) {
- type = config.getType();
- }
-
- return type;
- }
-
- /**
- * get Type to use based on supplied parameters.
- *
- * @param metadata metadata
- * @param config config
- * @return result
- */
- public static String getType(Map<String, Object> metadata, ElasticsearchReaderConfiguration config) {
-
- String type = null;
-
- if ( metadata != null && metadata.containsKey("type")) {
- type = (String) metadata.get("type");
- }
-
- if (type == null) {
- type = config.getTypes().get(0);
- }
-
-
- return type;
- }
-
- /**
- * get id to use based on supplied parameters.
- *
- * @param datum datum
- * @return result
- */
- public static String getId(StreamsDatum datum) {
-
- String id = datum.getId();
-
- Map<String, Object> metadata = datum.getMetadata();
-
- if ( id == null && metadata != null && metadata.containsKey("id")) {
- id = (String) datum.getMetadata().get("id");
- }
-
- return id;
- }
-
- /**
- * get id to use based on supplied parameters.
- *
- * @param metadata metadata
- * @return result
- */
- public static String getId(Map<String, Object> metadata) {
-
- return (String) metadata.get("id");
-
- }
-
- /**
- * get parent id to use based on supplied parameters.
- *
- * @param datum datum
- * @return result
- */
- static String getParent(StreamsDatum datum) {
-
- String parent = null;
-
- Map<String, Object> metadata = datum.getMetadata();
-
- if (metadata != null && metadata.containsKey("parent")) {
- parent = (String) datum.getMetadata().get("parent");
- }
-
- return parent;
- }
-
- /**
- * get routing id to use based on supplied parameters.
- *
- * @param datum datum
- * @return result
- */
- static String getRouting(StreamsDatum datum) {
-
- String routing = null;
-
- Map<String, Object> metadata = datum.getMetadata();
-
- if (metadata != null && metadata.containsKey("routing")) {
- routing = (String) datum.getMetadata().get("routing");
- }
-
- return routing;
- }
-
- /**
- * get JsonNode as Map.
- * @param node node
- * @return result
- */
- // TODO: move this to a utility package
- public static Map<String, Object> asMap(JsonNode node) {
-
- Iterator<Map.Entry<String, JsonNode>> iterator = node.fields();
- Map<String, Object> ret = new HashMap<>();
-
- Map.Entry<String, JsonNode> entry;
-
- while (iterator.hasNext()) {
- entry = iterator.next();
- if ( entry.getValue().asText() != null ) {
- ret.put(entry.getKey(), entry.getValue().asText());
- }
- }
-
- return ret;
- }
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistDeleter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistDeleter.java
deleted file mode 100644
index 3c4ebc9..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistDeleter.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * ElasticsearchPersistDeleter deletes documents from elasticsearch.
- */
-public class ElasticsearchPersistDeleter extends ElasticsearchPersistWriter implements StreamsPersistWriter {
-
- private static final String STREAMS_ID = ElasticsearchPersistDeleter.class.getCanonicalName();
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistDeleter.class);
-
- public ElasticsearchPersistDeleter() {
- super();
- }
-
- public ElasticsearchPersistDeleter(ElasticsearchWriterConfiguration config) {
- super(config);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void write(StreamsDatum streamsDatum) {
-
- if ( streamsDatum == null || streamsDatum.getDocument() == null) {
- return;
- }
-
- LOGGER.debug("Delete Document: {}", streamsDatum.getDocument());
-
- Map<String, Object> metadata = streamsDatum.getMetadata();
-
- LOGGER.debug("Delete Metadata: {}", metadata);
-
- String index = ElasticsearchMetadataUtil.getIndex(metadata, config);
- String type = ElasticsearchMetadataUtil.getType(metadata, config);
- String id = ElasticsearchMetadataUtil.getId(streamsDatum);
-
- try {
- delete(index, type, id);
- } catch (Throwable ex) {
- LOGGER.warn("Unable to Delete Document from ElasticSearch: {}", ex.getMessage());
- }
- }
-
- /**
- * Prepare and en-queue @see org.elasticsearch.action.delete.DeleteRequest
- * @param index index
- * @param type type
- * @param id id
- */
- public void delete(String index, String type, String id) {
- DeleteRequest deleteRequest;
-
- Objects.requireNonNull(index);
- Objects.requireNonNull(id);
- Objects.requireNonNull(type);
-
- // They didn't specify an ID, so we will create one for them.
- deleteRequest = new DeleteRequest()
- .index(index)
- .type(type)
- .id(id);
-
- add(deleteRequest);
-
- }
-
- /**
- * Enqueue DeleteRequest.
- * @param request request
- */
- public void add(DeleteRequest request) {
-
- Objects.requireNonNull(request);
- Objects.requireNonNull(request.index());
-
- // If our queue is larger than our flush threshold, then we should flush the queue.
- synchronized (this) {
- checkIndexImplications(request.index());
-
- bulkRequest.add(request);
-
- currentBatchItems.incrementAndGet();
-
- checkForFlush();
- }
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
deleted file mode 100644
index 27285f9..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch;
-
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Queues;
-import org.elasticsearch.search.SearchHit;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.Queue;
-import java.util.Random;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * ElasticsearchPersistReader reads documents from elasticsearch.
- */
-public class ElasticsearchPersistReader implements StreamsPersistReader, Serializable {
-
- public static final String STREAMS_ID = "ElasticsearchPersistReader";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistReader.class);
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ElasticsearchQuery elasticsearchQuery;
- private ElasticsearchReaderConfiguration config;
- private int threadPoolSize = 10;
- private ExecutorService executor;
- private ReadWriteLock lock = new ReentrantReadWriteLock();
- private Future<?> readerTask;
-
- public ElasticsearchPersistReader() {
- }
-
- public ElasticsearchPersistReader(ElasticsearchReaderConfiguration config) {
- this.config = config;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- //PersistReader methods
- @Override
- public void startStream() {
- LOGGER.debug("startStream");
- executor = Executors.newSingleThreadExecutor();
- readerTask = executor.submit(new ElasticsearchPersistReaderTask(this, elasticsearchQuery));
- }
-
- @Override
- public void prepare(Object configuration) {
- elasticsearchQuery = this.config == null ? new ElasticsearchQuery() : new ElasticsearchQuery(config);
- elasticsearchQuery.execute(configuration);
- persistQueue = constructQueue();
- }
-
- @Override
- public StreamsResultSet readAll() {
- return readCurrent();
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- try {
- lock.writeLock().lock();
- current = new StreamsResultSet(persistQueue);
- current.setCounter(new DatumStatusCounter());
- persistQueue = constructQueue();
- } finally {
- lock.writeLock().unlock();
- }
-
- return current;
-
- }
-
- //TODO - This just reads current records and does not adjust any queries
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return readCurrent();
- }
-
- //TODO - This just reads current records and does not adjust any queries
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return readCurrent();
- }
-
- //If we still have data in the queue, we are still running
- @Override
- public boolean isRunning() {
- return persistQueue.size() > 0 || (!readerTask.isDone() && !readerTask.isCancelled());
- }
-
- @Override
- public void cleanUp() {
- this.shutdownAndAwaitTermination(executor);
- LOGGER.info("PersistReader done");
- if ( elasticsearchQuery != null ) {
- elasticsearchQuery.cleanUp();
- }
- }
-
- //The locking may appear to be counter intuitive but we really don't care if multiple threads offer to the queue
- //as it is a synchronized queue. What we do care about is that we don't want to be offering to the current reference
- //if the queue is being replaced with a new instance
- protected void write(StreamsDatum entry) {
- boolean success;
- do {
- try {
- lock.readLock().lock();
- success = persistQueue.offer(entry);
- Thread.yield();
- } finally {
- lock.readLock().unlock();
- }
- }
- while (!success);
- }
-
- protected void shutdownAndAwaitTermination(ExecutorService pool) {
- pool.shutdown(); // Disable new tasks from being submitted
- try {
- // Wait a while for existing tasks to terminate
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- pool.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- LOGGER.error("Pool did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- pool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
- private Queue<StreamsDatum> constructQueue() {
- return Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
- }
-
- public static class ElasticsearchPersistReaderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistReaderTask.class);
-
- private ElasticsearchPersistReader reader;
- private ElasticsearchQuery query;
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- public ElasticsearchPersistReaderTask(ElasticsearchPersistReader reader, ElasticsearchQuery query) {
- this.reader = reader;
- this.query = query;
- }
-
- @Override
- public void run() {
-
- StreamsDatum item;
- while (query.hasNext()) {
- SearchHit hit = query.next();
- ObjectNode jsonObject;
- try {
- jsonObject = mapper.readValue(hit.getSourceAsString(), ObjectNode.class);
- item = new StreamsDatum(jsonObject, hit.getId());
- item.getMetadata().put("id", hit.getId());
- item.getMetadata().put("index", hit.getIndex());
- item.getMetadata().put("type", hit.getType());
- if ( hit.fields().containsKey("_timestamp")) {
- DateTime timestamp = new DateTime(((Long) hit.field("_timestamp").getValue()).longValue());
- item.setTimestamp(timestamp);
- }
- if ( hit.fields().containsKey("_parent")) {
- item.getMetadata().put("parent", hit.fields().get("_parent").value());
- }
- reader.write(item);
- } catch (IOException ex) {
- LOGGER.warn("Unable to process json source: ", hit.getSourceAsString());
- }
-
- }
- try {
- Thread.sleep(new Random().nextInt(100));
- } catch (InterruptedException ex) {
- LOGGER.warn("Thread interrupted", ex);
- }
-
- }
- }
-}
-
-
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java
deleted file mode 100644
index c8b95d6..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-
-import org.apache.commons.lang3.StringUtils;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * ElasticsearchPersistUpdater updates documents to elasticsearch.
- */
-public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter implements StreamsPersistWriter {
-
- private static final String STREAMS_ID = ElasticsearchPersistUpdater.class.getCanonicalName();
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistUpdater.class);
-
- public ElasticsearchPersistUpdater() {
- super();
- }
-
- public ElasticsearchPersistUpdater(ElasticsearchWriterConfiguration config) {
- super(config);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void write(StreamsDatum streamsDatum) {
-
- if (streamsDatum == null || streamsDatum.getDocument() == null) {
- return;
- }
-
- LOGGER.debug("Update Document: {}", streamsDatum.getDocument());
-
- Map<String, Object> metadata = streamsDatum.getMetadata();
-
- LOGGER.debug("Update Metadata: {}", metadata);
-
- String index = ElasticsearchMetadataUtil.getIndex(metadata, config);
- String type = ElasticsearchMetadataUtil.getType(metadata, config);
- String id = ElasticsearchMetadataUtil.getId(streamsDatum);
- String parent = ElasticsearchMetadataUtil.getParent(streamsDatum);
- String routing = ElasticsearchMetadataUtil.getRouting(streamsDatum);
-
- try {
-
- String docAsJson = docAsJson(streamsDatum.getDocument());
-
- LOGGER.debug("Attempt Update: ({},{},{},{},{}) {}", index, type, id, parent, routing, docAsJson);
-
- update(index, type, id, parent, routing, docAsJson);
-
- } catch (Throwable ex) {
- LOGGER.warn("Unable to Update Document in ElasticSearch: {}", ex.getMessage());
- }
- }
-
- /**
- * Prepare and en-queue.
- * @see org.elasticsearch.action.update.UpdateRequest
- * @param indexName indexName
- * @param type type
- * @param id id
- * @param parent parent
- * @param routing routing
- * @param json json
- */
- public void update(String indexName, String type, String id, String parent, String routing, String json) {
- UpdateRequest updateRequest;
-
- Objects.requireNonNull(id);
- Objects.requireNonNull(json);
-
- // They didn't specify an ID, so we will create one for them.
- updateRequest = new UpdateRequest()
- .index(indexName)
- .type(type)
- .id(id)
- .doc(json);
-
- if (StringUtils.isNotBlank(parent)) {
- updateRequest = updateRequest.parent(parent);
- }
-
- if (StringUtils.isNotBlank(routing)) {
- updateRequest = updateRequest.routing(routing);
- }
-
- // add fields
- //updateRequest.docAsUpsert(true);
-
- add(updateRequest);
-
- }
-
- /**
- * Enqueue UpdateRequest.
- * @param request request
- */
- public void add(UpdateRequest request) {
-
- Objects.requireNonNull(request);
- Objects.requireNonNull(request.index());
-
- // If our queue is larger than our flush threshold, then we should flush the queue.
- synchronized (this) {
- checkIndexImplications(request.index());
-
- bulkRequest.add(request);
-
- currentBatchBytes.addAndGet(request.doc().source().length());
- currentBatchItems.incrementAndGet();
-
- checkForFlush();
- }
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
deleted file mode 100644
index d4a62da..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ /dev/null
@@ -1,636 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.common.settings.Settings;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.text.DecimalFormat;
-import java.text.NumberFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * ElasticsearchPersistUpdater updates documents to elasticsearch.
- */
-public class ElasticsearchPersistWriter implements StreamsPersistWriter, Serializable {
-
- public static final String STREAMS_ID = ElasticsearchPersistWriter.class.getCanonicalName();
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistWriter.class);
- private static final NumberFormat MEGABYTE_FORMAT = new DecimalFormat("#.##");
- private static final NumberFormat NUMBER_FORMAT = new DecimalFormat("###,###,###,###");
- private static final Long DEFAULT_BULK_FLUSH_THRESHOLD = 5L * 1024L * 1024L;
- private static final int DEFAULT_BATCH_SIZE = 100;
- //ES defaults its bulk index queue to 50 items. We want to be under this on our backoff so set this to 1/2 ES default
- //at a batch size as configured here.
- private static final long WAITING_DOCS_LIMIT = DEFAULT_BATCH_SIZE * 25;
- //A document should have to wait no more than 10s to get flushed
- private static final long DEFAULT_MAX_WAIT = 10000;
-
- protected static final ObjectMapper OBJECT_MAPPER = StreamsJacksonMapper.getInstance();
-
- protected final List<String> affectedIndexes = new ArrayList<>();
-
- protected final ElasticsearchClientManager manager;
- protected final ElasticsearchWriterConfiguration config;
-
- protected BulkRequestBuilder bulkRequest;
-
- private boolean veryLargeBulk = false; // by default this setting is set to false
- private long flushThresholdsRecords = DEFAULT_BATCH_SIZE;
- private long flushThresholdBytes = DEFAULT_BULK_FLUSH_THRESHOLD;
-
- private long flushThresholdTime = DEFAULT_MAX_WAIT;
- private long lastFlush = new Date().getTime();
- private Timer timer = new Timer();
-
-
- private final AtomicInteger batchesSent = new AtomicInteger(0);
- private final AtomicInteger batchesResponded = new AtomicInteger(0);
-
- protected final AtomicLong currentBatchItems = new AtomicLong(0);
- protected final AtomicLong currentBatchBytes = new AtomicLong(0);
-
- private final AtomicLong totalSent = new AtomicLong(0);
- private final AtomicLong totalSeconds = new AtomicLong(0);
- private final AtomicLong totalOk = new AtomicLong(0);
- private final AtomicLong totalFailed = new AtomicLong(0);
- private final AtomicLong totalSizeInBytes = new AtomicLong(0);
-
- public ElasticsearchPersistWriter() {
- this(new ComponentConfigurator<>(ElasticsearchWriterConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("elasticsearch")));
- }
-
- public ElasticsearchPersistWriter(ElasticsearchWriterConfiguration config) {
- this(config, ElasticsearchClientManager.getInstance(config));
- }
-
- /**
- * ElasticsearchPersistWriter constructor.
- * @param config config
- * @param manager manager
- */
- public ElasticsearchPersistWriter(ElasticsearchWriterConfiguration config, ElasticsearchClientManager manager) {
- this.config = config;
- this.manager = manager;
- this.bulkRequest = this.manager.client().prepareBulk();
- }
-
- public long getBatchesSent() {
- return this.batchesSent.get();
- }
-
- public long getBatchesResponded() {
- return batchesResponded.get();
- }
-
- public long getFlushThresholdsRecords() {
- return this.flushThresholdsRecords;
- }
-
- public long getFlushThresholdBytes() {
- return this.flushThresholdBytes;
- }
-
- public long getFlushThreasholdMaxTime() {
- return this.flushThresholdTime;
- }
-
- public void setFlushThresholdRecords(long val) {
- this.flushThresholdsRecords = val;
- }
-
- public void setFlushThresholdBytes(long val) {
- this.flushThresholdBytes = val;
- }
-
- public void setFlushThreasholdMaxTime(long val) {
- this.flushThresholdTime = val;
- }
-
- public void setVeryLargeBulk(boolean veryLargeBulk) {
- this.veryLargeBulk = veryLargeBulk;
- }
-
- private long getLastFlush() {
- return this.lastFlush;
- }
-
- public long getTotalOutstanding() {
- return this.totalSent.get() - (this.totalFailed.get() + this.totalOk.get());
- }
-
- public long getTotalSent() {
- return this.totalSent.get();
- }
-
- public long getTotalOk() {
- return this.totalOk.get();
- }
-
- public long getTotalFailed() {
- return this.totalFailed.get();
- }
-
- public long getTotalSizeInBytes() {
- return this.totalSizeInBytes.get();
- }
-
- public long getTotalSeconds() {
- return this.totalSeconds.get();
- }
-
- public List<String> getAffectedIndexes() {
- return this.affectedIndexes;
- }
-
- public boolean isConnected() {
- return (this.manager.client() != null);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void write(StreamsDatum streamsDatum) {
-
- if (streamsDatum == null || streamsDatum.getDocument() == null) {
- return;
- }
-
- checkForBackOff();
-
- LOGGER.debug("Write Document: {}", streamsDatum.getDocument());
-
- Map<String, Object> metadata = streamsDatum.getMetadata();
-
- LOGGER.debug("Write Metadata: {}", metadata);
-
- String index = ElasticsearchMetadataUtil.getIndex(metadata, config);
- String type = ElasticsearchMetadataUtil.getType(metadata, config);
- String id = ElasticsearchMetadataUtil.getId(streamsDatum);
- String parent = ElasticsearchMetadataUtil.getParent(streamsDatum);
- String routing = ElasticsearchMetadataUtil.getRouting(streamsDatum);
-
- try {
- streamsDatum = appendMetadata(streamsDatum);
- String docAsJson = docAsJson(streamsDatum.getDocument());
- add(index, type, id, parent, routing,
- streamsDatum.getTimestamp() == null ? Long.toString(DateTime.now().getMillis()) : Long.toString(streamsDatum.getTimestamp().getMillis()),
- docAsJson);
- } catch (Throwable ex) {
- LOGGER.warn("Unable to Write Datum to ElasticSearch: {}", ex.getMessage());
- }
- }
-
- protected String docAsJson(Object streamsDocument) throws IOException {
- return (streamsDocument instanceof String) ? streamsDocument.toString() : OBJECT_MAPPER.writeValueAsString(streamsDocument);
- }
-
- protected StreamsDatum appendMetadata(StreamsDatum streamsDatum) throws IOException {
-
- String docAsJson = (streamsDatum.getDocument() instanceof String) ? streamsDatum.getDocument().toString() : OBJECT_MAPPER.writeValueAsString(streamsDatum.getDocument());
-
- if (streamsDatum.getMetadata() == null || streamsDatum.getMetadata().size() == 0) {
- return streamsDatum;
- } else {
- ObjectNode node = (ObjectNode)OBJECT_MAPPER.readTree(docAsJson);
- node.put("_metadata", OBJECT_MAPPER.readTree(OBJECT_MAPPER.writeValueAsBytes(streamsDatum.getMetadata())));
- streamsDatum.setDocument(OBJECT_MAPPER.writeValueAsString(node));
- return streamsDatum;
- }
- }
-
- @Override
- public void cleanUp() {
-
- try {
-
- LOGGER.debug("cleanUp started");
-
- // before they close, check to ensure that
- flushInternal();
-
- LOGGER.debug("flushInternal completed");
-
- waitToCatchUp(0, 5 * 60 * 1000);
-
- LOGGER.debug("waitToCatchUp completed");
-
- } catch (Throwable ex) {
- // this line of code should be logically unreachable.
- LOGGER.warn("This is unexpected: {}", ex);
- } finally {
-
- if (veryLargeBulk) {
- resetRefreshInterval();
- }
-
- if ( config.getRefresh() ) {
- refreshIndexes();
- LOGGER.debug("refreshIndexes completed");
- }
-
- LOGGER.debug("Closed ElasticSearch Writer: Ok[{}] Failed[{}] Orphaned[{}]",
- this.totalOk.get(), this.totalFailed.get(), this.getTotalOutstanding());
- timer.cancel();
-
- LOGGER.debug("cleanUp completed");
- }
- }
-
- private void resetRefreshInterval() {
- for (String indexName : this.affectedIndexes) {
-
- if (this.veryLargeBulk) {
- LOGGER.debug("Resetting our Refresh Interval: {}", indexName);
- // They are in 'very large bulk' mode and the process is finished. We now want to turn the
- // refreshing back on.
- UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
- updateSettingsRequest.settings(Settings.settingsBuilder().put("refresh_interval", "5s"));
-
- // submit to ElasticSearch
- this.manager.client()
- .admin()
- .indices()
- .updateSettings(updateSettingsRequest)
- .actionGet();
- }
- }
- }
-
- private void refreshIndexes() {
-
- for (String indexName : this.affectedIndexes) {
-
- if (config.getRefresh()) {
- LOGGER.debug("Refreshing ElasticSearch index: {}", indexName);
- this.manager.client()
- .admin()
- .indices()
- .prepareRefresh(indexName)
- .execute()
- .actionGet();
- }
- }
- }
-
- private synchronized void flushInternal() {
- // we do not have a working bulk request, we can just exit here.
- if (this.bulkRequest == null || this.currentBatchItems.get() == 0) {
- return;
- }
-
- // wait for one minute to catch up if it needs to
- waitToCatchUp(5, 60 * 1000);
-
- // call the flush command.
- flush(this.bulkRequest, this.currentBatchItems.get(), this.currentBatchBytes.get());
-
- // reset the current batch statistics
- this.currentBatchItems.set(0);
- this.currentBatchBytes.set(0);
-
- // reset our bulk request builder
- this.bulkRequest = this.manager.client().prepareBulk();
- }
-
- private synchronized void waitToCatchUp(int batchThreshold, int timeOutThresholdInMS) {
- int counter = 0;
- // If we still have 5 batches outstanding, we need to give it a minute to catch up
- while (this.getBatchesSent() - this.getBatchesResponded() > batchThreshold && counter < timeOutThresholdInMS) {
- try {
- Thread.yield();
- Thread.sleep(1);
- counter++;
- } catch (InterruptedException ie) {
- LOGGER.warn("Catchup was interrupted. Data may be lost");
- return;
- }
- }
- }
-
- private void checkForBackOff() {
- try {
- if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT) {
- /*
- * Author:
- * Smashew
- *
- * Date:
- * 2013-10-20
- *
- * Note:
- * With the information that we have on hand. We need to develop a heuristic
- * that will determine when the cluster is having a problem indexing records
- * by telling it to pause and wait for it to catch back up. A
- *
- * There is an impact to us, the caller, whenever this happens as well. Items
- * that are not yet fully indexed by the server sit in a queue, on the client
- * that can cause the heap to overflow. This has been seen when re-indexing
- * large amounts of data to a small cluster. The "deletes" + "indexes" can
- * cause the server to have many 'outstandingItems" in queue. Running this
- * software with large amounts of data, on a small cluster, can re-create
- * this problem.
- *
- * DO NOT DELETE THESE LINES
- ****************************************************************************/
-
- // wait for the flush to catch up. We are going to cap this at
- int count = 0;
- while (this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500) {
- Thread.sleep(10);
- }
- if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT) {
- LOGGER.warn("Even after back-off there are {} items still in queue.", this.getTotalOutstanding());
- }
- }
- } catch (Exception ex) {
- LOGGER.warn("We were broken from our loop: {}", ex.getMessage());
- }
- }
-
- /**
- * add based on supplied parameters.
- * @param indexName indexName
- * @param type type
- * @param id id
- * @param ts ts
- * @param json json
- */
- public void add(String indexName, String type, String id, String ts, String json) {
- add(indexName, type, id, null, null, ts, json);
- }
-
- /**
- * add based on supplied parameters.
- * @param indexName indexName
- * @param type type
- * @param id id
- * @param routing routing
- * @param ts ts
- * @param json json
- */
- public void add(String indexName, String type, String id, String parent, String routing, String ts, String json) {
-
- // make sure that these are not null
- Objects.requireNonNull(indexName);
- Objects.requireNonNull(type);
- Objects.requireNonNull(json);
-
- IndexRequestBuilder indexRequestBuilder = manager.client()
- .prepareIndex(indexName, type)
- .setSource(json);
-
- // / They didn't specify an ID, so we will create one for them.
- if (id != null) {
- indexRequestBuilder.setId(id);
- }
- if (ts != null) {
- indexRequestBuilder.setTimestamp(ts);
- }
- if (parent != null) {
- indexRequestBuilder.setParent(parent);
- }
- if (routing != null) {
- indexRequestBuilder.setRouting(routing);
- }
- add(indexRequestBuilder.request());
- }
-
- protected void add(IndexRequest request) {
-
- Objects.requireNonNull(request);
- Objects.requireNonNull(request.index());
-
- // If our queue is larger than our flush threshold, then we should flush the queue.
- synchronized (this) {
- checkIndexImplications(request.index());
-
- bulkRequest.add(request);
-
- this.currentBatchBytes.addAndGet(request.source().length());
- this.currentBatchItems.incrementAndGet();
-
- checkForFlush();
- }
- }
-
- protected void checkForFlush() {
- synchronized (this) {
- if (this.currentBatchBytes.get() >= this.flushThresholdBytes
- ||
- this.currentBatchItems.get() >= this.flushThresholdsRecords
- ||
- new Date().getTime() - this.lastFlush >= this.flushThresholdTime) {
- // We should flush
- flushInternal();
- }
- }
- }
-
- protected void checkIndexImplications(String indexName) {
- // We need this to be safe across all writers that are currently being executed
- synchronized (ElasticsearchPersistWriter.class) {
-
- // this will be common if we have already verified the index.
- if (this.affectedIndexes.contains(indexName)) {
- return;
- }
-
- // create the index if it is missing
- createIndexIfMissing(indexName);
-
- // we haven't log this index.
- this.affectedIndexes.add(indexName);
-
- }
- }
-
- protected void disableRefresh() {
-
- for (String indexName : this.affectedIndexes) {
- // They are in 'very large bulk' mode we want to turn off refreshing the index.
- // Create a request then add the setting to tell it to stop refreshing the interval
- UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
- updateSettingsRequest.settings(Settings.settingsBuilder().put("refresh_interval", -1));
-
- // submit to ElasticSearch
- this.manager.client()
- .admin()
- .indices()
- .updateSettings(updateSettingsRequest)
- .actionGet();
- }
- }
-
- /**
- * createIndexIfMissing
- * @param indexName indexName
- */
- public void createIndexIfMissing(String indexName) {
- // Synchronize this on a static class level
- if (!this.manager.client()
- .admin()
- .indices()
- .exists(new IndicesExistsRequest(indexName))
- .actionGet()
- .isExists()) {
- // It does not exist... So we are going to need to create the index.
- // we are going to assume that the 'templates' that we have loaded into
- // elasticsearch are sufficient to ensure the index is being created properly.
- CreateIndexResponse response = this.manager.client().admin().indices().create(new CreateIndexRequest(indexName)).actionGet();
-
- if (response.isAcknowledged()) {
- LOGGER.info("Index Created: {}", indexName);
- } else {
- LOGGER.error("Index {} did not exist. While attempting to create the index from stored ElasticSearch Templates we were unable to get an acknowledgement.", indexName);
- LOGGER.error("Error Message: {}", response.toString());
- throw new RuntimeException("Unable to create index " + indexName);
- }
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.veryLargeBulk = config.getBulk() == null
- ? Boolean.FALSE
- : config.getBulk();
-
- this.flushThresholdsRecords = config.getBatchSize() == null
- ? DEFAULT_BATCH_SIZE
- : (int)(config.getBatchSize().longValue());
-
- this.flushThresholdTime = config.getMaxTimeBetweenFlushMs() != null && config.getMaxTimeBetweenFlushMs() > 0
- ? config.getMaxTimeBetweenFlushMs()
- : DEFAULT_MAX_WAIT;
-
- this.flushThresholdBytes = config.getBatchBytes() == null
- ? DEFAULT_BULK_FLUSH_THRESHOLD
- : config.getBatchBytes();
-
- timer.scheduleAtFixedRate(new TimerTask() {
- public void run() {
- checkForFlush();
- }
- }, this.flushThresholdTime, this.flushThresholdTime);
-
- if ( veryLargeBulk ) {
- disableRefresh();
- }
- }
-
- private void flush(final BulkRequestBuilder bulkRequest, final Long sent, final Long sizeInBytes) {
- LOGGER.debug("Writing to ElasticSearch: Items[{}] Size[{} mb]", sent, MEGABYTE_FORMAT.format(sizeInBytes / (double) (1024 * 1024)));
-
-
- // record the last time we flushed the index
- this.lastFlush = new Date().getTime();
-
- // add the totals
- this.totalSent.addAndGet(sent);
-
- // add the total number of batches sent
- this.batchesSent.incrementAndGet();
-
- try {
- bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
- public void onResponse(BulkResponse bulkItemResponses) {
- batchesResponded.incrementAndGet();
- updateTotals(bulkItemResponses, sent, sizeInBytes);
- }
-
- public void onFailure(Throwable throwable) {
- batchesResponded.incrementAndGet();
- throwable.printStackTrace();
- }
- });
- } catch (Throwable ex) {
- LOGGER.error("There was an error sending the batch: {}", ex.getMessage());
- }
- }
-
- private void updateTotals(final BulkResponse bulkItemResponses, final Long sent, final Long sizeInBytes) {
- long failed = 0;
- long passed = 0;
- long millis = bulkItemResponses.getTookInMillis();
-
- // keep track of the number of totalFailed and items that we have totalOk.
- for (BulkItemResponse resp : bulkItemResponses.getItems()) {
- if (resp == null || resp.isFailed()) {
- failed++;
- LOGGER.debug("{} ({},{},{}) failed: {}", resp.getOpType(), resp.getIndex(), resp.getType(), resp.getId(), resp.getFailureMessage());
- } else {
- passed++;
- }
- }
-
- if (failed > 0) {
- LOGGER.warn("Bulk Uploading had {} failures of {}", failed, sent);
- }
-
- this.totalOk.addAndGet(passed);
- this.totalFailed.addAndGet(failed);
- this.totalSeconds.addAndGet(millis / 1000);
- this.totalSizeInBytes.addAndGet(sizeInBytes);
-
- if (sent != (passed + failed)) {
- LOGGER.error("Count MisMatch: Sent[{}] Passed[{}] Failed[{}]", sent, passed, failed);
- }
-
- LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
- MEGABYTE_FORMAT.format(sizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(passed), NUMBER_FORMAT.format(failed), NUMBER_FORMAT.format(millis),
- MEGABYTE_FORMAT.format((double) totalSizeInBytes.get() / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
- }
-
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
deleted file mode 100644
index 29dafd9..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.sort.SortBuilders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Helper for building, querying, and paging an elasticsearch query.
- */
-public class ElasticsearchQuery implements Iterable<SearchHit>, Iterator<SearchHit>, Serializable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchQuery.class);
- private static final int SCROLL_POSITION_NOT_INITIALIZED = -3;
-
- private ElasticsearchClientManager elasticsearchClientManager;
- private ElasticsearchReaderConfiguration config;
- private List<String> indexes = new ArrayList<>();
- private List<String> types = new ArrayList<>();
- private int limit = 1000 * 1000 * 1000; // we are going to set the default limit very high to 1bil
- private int batchSize = 100;
- private String scrollTimeout = "5m";
- private org.elasticsearch.index.query.QueryBuilder queryBuilder;
- private SearchRequestBuilder search;
- private SearchResponse scrollResp;
- private int scrollPositionInScroll = SCROLL_POSITION_NOT_INITIALIZED;
- private SearchHit next = null;
- private long totalHits = 0;
- private long totalRead = 0;
-
- private StreamsJacksonMapper mapper = StreamsJacksonMapper.getInstance();
-
- /**
- * ElasticsearchQuery constructor - resolves ElasticsearchReaderConfiguration from JVM 'elasticsearch'.
- */
- public ElasticsearchQuery() {
- this(new ComponentConfigurator<>(ElasticsearchReaderConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("elasticsearch")));
- }
-
- /**
- * ElasticsearchQuery constructor - uses provided ElasticsearchReaderConfiguration.
- */
- public ElasticsearchQuery(ElasticsearchReaderConfiguration config) {
- this.config = config;
- this.elasticsearchClientManager = ElasticsearchClientManager.getInstance(config);
- this.indexes.addAll(config.getIndexes());
- this.types.addAll(config.getTypes());
- this.scrollTimeout = config.getScrollTimeout();
- }
-
- public long getHitCount() {
- return this.search == null ? 0 : this.totalHits;
- }
-
- public long getReadCount() {
- return this.totalRead;
- }
-
- public double getReadPercent() {
- return (double) this.getReadCount() / (double) this.getHitCount();
- }
-
- public long getRemainingCount() {
- return this.totalRead - this.totalHits;
- }
-
- public void setBatchSize(int batchSize) {
- this.batchSize = batchSize;
- }
-
- public void setScrollTimeout(String scrollTimeout) {
- this.scrollTimeout = scrollTimeout;
- }
-
- public void setQueryBuilder(QueryBuilder queryBuilder) {
- this.queryBuilder = queryBuilder;
- }
-
- /**
- * execute ElasticsearchQuery.
- * @param obj deprecated
- */
- public void execute(Object obj) {
-
- // If we haven't already set up the search, then set up the search.
- if (search == null) {
-
- search = elasticsearchClientManager.client()
- .prepareSearch(indexes.toArray(new String[0]))
- .setSearchType(SearchType.SCAN)
- .setExplain(true)
- .addField("*")
- .setFetchSource(true)
- .setSize(batchSize)
- .setScroll(scrollTimeout)
- .addField("_timestamp");
-
- LOGGER.debug("Search source: " + search.toString());
-
- String searchJson;
- if ( config.getSearch() != null ) {
- LOGGER.debug("Have config in Reader: " + config.getSearch().toString());
-
- try {
- searchJson = mapper.writeValueAsString(config.getSearch());
- LOGGER.debug("Extra source: " + searchJson);
- search = search.setExtraSource(searchJson);
-
- } catch (JsonProcessingException ex) {
- LOGGER.warn("Could not apply _search supplied by config", ex.getMessage());
- }
-
-
- }
-
- LOGGER.debug("Final Search: " + search.internalBuilder().toString());
-
- if (this.queryBuilder != null) {
- search = search.setQuery(this.queryBuilder);
- }
-
- // If the types are null, then don't specify a type
- if (this.types != null && this.types.size() > 0) {
- search = search.setTypes(types.toArray(new String[0]));
- }
-
- // TODO: Replace when all clusters are upgraded past 0.90.4 so we can implement a RANDOM scroll.
- boolean random = false;
- if (random) {
- search = search.addSort(SortBuilders.scriptSort(new Script("random()"), "number"));
- }
- }
-
- // We don't have a scroll, we need to create a scroll
- if (scrollResp == null) {
- scrollResp = search.execute().actionGet();
- LOGGER.trace(search.toString());
- }
- }
-
- //Iterable methods
- @Override
- public Iterator<SearchHit> iterator() {
- return this;
- }
-
- //Iterator methods
- @Override
- public SearchHit next() {
- return this.next;
- }
-
- @Override
- public boolean hasNext() {
- calcNext();
- return hasRecords();
- }
-
- /**
- * shift to next page of scroll.
- */
- public void calcNext() {
- try {
- // We have exhausted our scroll create another scroll.
- if (scrollPositionInScroll == SCROLL_POSITION_NOT_INITIALIZED || scrollPositionInScroll >= scrollResp.getHits().getHits().length) {
- // reset the scroll position
- scrollPositionInScroll = 0;
-
- // get the next hits of the scroll
- scrollResp = elasticsearchClientManager.client()
- .prepareSearchScroll(scrollResp.getScrollId())
- .setScroll(scrollTimeout)
- .execute()
- .actionGet();
-
- this.totalHits = scrollResp.getHits().getTotalHits();
- }
-
- // If this scroll has 0 items then we set the scroll position to -1
- // letting the iterator know that we are done.
- if (scrollResp.getHits().getTotalHits() == 0 || scrollResp.getHits().getHits().length == 0) {
- scrollPositionInScroll = -1;
- } else {
- // get the next record
- next = scrollResp.getHits().getAt(scrollPositionInScroll);
-
- // Increment our counters
- scrollPositionInScroll += 1;
- totalRead += 1;
- }
- } catch (Exception ex) {
- LOGGER.error("Unexpected scrolling error: {}", ex.getMessage());
- scrollPositionInScroll = -1;
- next = null;
- }
- }
-
- public void remove() {
- }
-
- public void cleanUp() {
- }
-
- protected boolean isCompleted() {
- return totalRead >= this.limit && hasRecords();
- }
-
- protected boolean hasRecords() {
- return scrollPositionInScroll != -1 && (!(this.totalRead > this.limit));
- }
-
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/DatumFromMetadataAsDocumentProcessor.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/DatumFromMetadataAsDocumentProcessor.java
deleted file mode 100644
index d025e14..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/DatumFromMetadataAsDocumentProcessor.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.processor;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.elasticsearch.ElasticsearchClientManager;
-import org.apache.streams.elasticsearch.ElasticsearchMetadataUtil;
-import org.apache.streams.elasticsearch.ElasticsearchReaderConfiguration;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.typesafe.config.Config;
-import org.elasticsearch.action.get.GetRequestBuilder;
-import org.elasticsearch.action.get.GetResponse;
-import org.joda.time.DateTime;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Uses index and type in metadata map stored in datum document to populate current document into datums.
- */
-public class DatumFromMetadataAsDocumentProcessor implements StreamsProcessor, Serializable {
-
- private static final String STREAMS_ID = "DatumFromMetadataProcessor";
-
- private ElasticsearchClientManager elasticsearchClientManager;
- private ElasticsearchReaderConfiguration config;
-
- private ObjectMapper mapper;
-
- public DatumFromMetadataAsDocumentProcessor() {
- this.config = new ComponentConfigurator<>(ElasticsearchReaderConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("elasticsearch"));
- }
-
- public DatumFromMetadataAsDocumentProcessor(Config config) {
- this.config = new ComponentConfigurator<>(ElasticsearchReaderConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("elasticsearch"));
- }
-
- public DatumFromMetadataAsDocumentProcessor(ElasticsearchReaderConfiguration config) {
- this.config = config;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- List<StreamsDatum> result = new ArrayList<>();
-
- ObjectNode metadataObjectNode;
- try {
- metadataObjectNode = mapper.readValue((String) entry.getDocument(), ObjectNode.class);
- } catch (IOException ex) {
- return result;
- }
-
- Map<String, Object> metadata = ElasticsearchMetadataUtil.asMap(metadataObjectNode);
-
- if (entry.getMetadata() == null) {
- return result;
- }
-
- String index = ElasticsearchMetadataUtil.getIndex(metadata, config);
- String type = ElasticsearchMetadataUtil.getType(metadata, config);
- String id = ElasticsearchMetadataUtil.getId(metadata);
-
- GetRequestBuilder getRequestBuilder = elasticsearchClientManager.client().prepareGet(index, type, id);
- getRequestBuilder.setFields("*", "_timestamp");
- getRequestBuilder.setFetchSource(true);
- GetResponse getResponse = getRequestBuilder.get();
-
- if ( getResponse == null || !getResponse.isExists() || getResponse.isSourceEmpty()) {
- return result;
- }
-
- entry.setDocument(getResponse.getSource());
- if ( getResponse.getField("_timestamp") != null) {
- DateTime timestamp = new DateTime(((Long) getResponse.getField("_timestamp").getValue()).longValue());
- entry.setTimestamp(timestamp);
- }
-
- result.add(entry);
-
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.elasticsearchClientManager = ElasticsearchClientManager.getInstance(config);
- mapper = StreamsJacksonMapper.getInstance();
- }
-
- @Override
- public void cleanUp() {
- //
- }
-
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/DatumFromMetadataProcessor.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/DatumFromMetadataProcessor.java
deleted file mode 100644
index 8d6f93b..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/DatumFromMetadataProcessor.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.processor;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.elasticsearch.ElasticsearchClientManager;
-import org.apache.streams.elasticsearch.ElasticsearchMetadataUtil;
-import org.apache.streams.elasticsearch.ElasticsearchReaderConfiguration;
-
-import com.typesafe.config.Config;
-import org.elasticsearch.action.get.GetRequestBuilder;
-import org.elasticsearch.action.get.GetResponse;
-import org.joda.time.DateTime;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Uses index and type in metadata to populate current document into datums.
- */
-public class DatumFromMetadataProcessor implements StreamsProcessor, Serializable {
-
- private static final String STREAMS_ID = "DatumFromMetadataProcessor";
-
- private ElasticsearchClientManager elasticsearchClientManager;
- private ElasticsearchReaderConfiguration config;
-
- public DatumFromMetadataProcessor() {
- this.config = new ComponentConfigurator<>(ElasticsearchReaderConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("elasticsearch"));
- }
-
- public DatumFromMetadataProcessor(Config config) {
- this.config = new ComponentConfigurator<>(ElasticsearchReaderConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("elasticsearch"));
- }
-
- public DatumFromMetadataProcessor(ElasticsearchReaderConfiguration config) {
- this.config = config;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- List<StreamsDatum> result = new ArrayList<>();
-
- if (entry == null || entry.getMetadata() == null) {
- return result;
- }
-
- Map<String, Object> metadata = entry.getMetadata();
-
- String index = ElasticsearchMetadataUtil.getIndex(metadata, config);
- String type = ElasticsearchMetadataUtil.getType(metadata, config);
- String id = ElasticsearchMetadataUtil.getId(entry);
-
- GetRequestBuilder getRequestBuilder = elasticsearchClientManager.client().prepareGet(index, type, id);
- getRequestBuilder.setFields("*", "_timestamp");
- getRequestBuilder.setFetchSource(true);
- GetResponse getResponse = getRequestBuilder.get();
-
- if ( getResponse == null || !getResponse.isExists() || getResponse.isSourceEmpty() ) {
- return result;
- }
-
- entry.setDocument(getResponse.getSource());
- if ( getResponse.getField("_timestamp") != null) {
- DateTime timestamp = new DateTime(((Long) getResponse.getField("_timestamp").getValue()).longValue());
- entry.setTimestamp(timestamp);
- }
-
- result.add(entry);
-
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.elasticsearchClientManager = ElasticsearchClientManager.getInstance(config);
-
- }
-
- @Override
- public void cleanUp() {
- this.elasticsearchClientManager.client().close();
- }
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/DocumentToMetadataProcessor.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/DocumentToMetadataProcessor.java
deleted file mode 100644
index f1dc04a..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/DocumentToMetadataProcessor.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.elasticsearch.ElasticsearchMetadataUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Moves a json representation of metadata out of the document to the metadata field.
- *
- * <p/>
- * This is useful if you have a list of document metadata references in the document field,
- * for example loaded from a file, and need them in the metadata field.
- */
-public class DocumentToMetadataProcessor implements StreamsProcessor, Serializable {
-
- private static final String STREAMS_ID = "DatumFromMetadataProcessor";
-
- private ObjectMapper mapper;
-
- private static final Logger LOGGER = LoggerFactory.getLogger(DocumentToMetadataProcessor.class);
-
- public DocumentToMetadataProcessor() {
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- List<StreamsDatum> result = new ArrayList<>();
-
- Object object = entry.getDocument();
- ObjectNode metadataObjectNode;
- try {
- String docAsJson = (object instanceof String) ? object.toString() : mapper.writeValueAsString(object);
- metadataObjectNode = mapper.readValue(docAsJson, ObjectNode.class);
- } catch (Throwable ex) {
- LOGGER.warn("Exception: %s", ex.getMessage());
- return result;
- }
-
- Map<String, Object> metadata = ElasticsearchMetadataUtil.asMap(metadataObjectNode);
-
- if ( metadata == null ) {
- return result;
- }
-
- entry.setMetadata(metadata);
-
- result.add(entry);
-
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- mapper = StreamsJacksonMapper.getInstance();
- }
-
- @Override
- public void cleanUp() {
- mapper = null;
- }
-
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/MetadataFromDocumentProcessor.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/MetadataFromDocumentProcessor.java
deleted file mode 100644
index 4bed3d4..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/MetadataFromDocumentProcessor.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Examines document to derive metadata fields.
- *
- * </p>
- * This is useful if you have a document with a populated 'id', and 'verb' or 'objectType' fields you want
- * to use as _id and _type respectively when indexing.
- */
-public class MetadataFromDocumentProcessor implements StreamsProcessor, Serializable {
-
- public static final String STREAMS_ID = "MetadataFromDocumentProcessor";
-
- private ObjectMapper mapper;
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MetadataFromDocumentProcessor.class);
-
- public MetadataFromDocumentProcessor() {
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- if ( mapper == null ) {
- mapper = StreamsJacksonMapper.getInstance();
- }
-
- List<StreamsDatum> result = new ArrayList<>();
-
- Map<String, Object> metadata = entry.getMetadata();
- if ( metadata == null ) {
- metadata = new HashMap<>();
- }
-
- String id = null;
- String type = null;
-
- Object document = entry.getDocument();
- ObjectNode objectNode = null;
- if ( document instanceof String) {
- try {
- objectNode = mapper.readValue((String) document, ObjectNode.class);
- } catch (IOException ex) {
- LOGGER.warn("Can't deserialize to determine metadata", ex);
- }
- } else {
- try {
- objectNode = mapper.convertValue(document, ObjectNode.class);
- } catch (Exception ex) {
- LOGGER.warn("Can't deserialize to determine metadata", ex);
- }
- }
- if ( objectNode != null ) {
- if (objectNode.has("id")) {
- id = objectNode.get("id").textValue();
- }
- if (objectNode.has("verb")) {
- type = objectNode.get("verb").textValue();
- }
- if (objectNode.has("objectType")) {
- type = objectNode.get("objectType").textValue();
- }
- }
-
- if (StringUtils.isNotEmpty(id) ) {
- metadata.put("id", id);
- }
- if (StringUtils.isNotEmpty(type) ) {
- metadata.put("type", type);
- }
-
- entry.setId(id);
- entry.setMetadata(metadata);
-
- result.add(entry);
-
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- mapper = StreamsJacksonMapper.getInstance();
- }
-
- @Override
- public void cleanUp() {
- mapper = null;
- }
-
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/PercolateTagProcessor.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/PercolateTagProcessor.java
deleted file mode 100644
index 8e82c16..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/processor/PercolateTagProcessor.java
+++ /dev/null
@@ -1,371 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.elasticsearch.ElasticsearchClientManager;
-import org.apache.streams.elasticsearch.ElasticsearchConfiguration;
-import org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.percolate.PercolateRequestBuilder;
-import org.elasticsearch.action.percolate.PercolateResponse;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.QueryStringQueryBuilder;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHits;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.Set;
-
-/**
- * References:
- * Some helpful references to help
- * Purpose URL
- * ------------- ----------------------------------------------------------------
- * [Status Codes] http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
- * [Test Cases] http://greenbytes.de/tech/tc/httpredirects/
- * [t.co behavior] https://dev.twitter.com/docs/tco-redirection-behavior
- */
-
-public class PercolateTagProcessor implements StreamsProcessor {
-
- public static final String STREAMS_ID = "PercolateTagProcessor";
- private static final Logger LOGGER = LoggerFactory.getLogger(PercolateTagProcessor.class);
- private static final String DEFAULT_PERCOLATE_FIELD = "_all";
-
- private ObjectMapper mapper;
-
- protected Queue<StreamsDatum> inQueue;
- protected Queue<StreamsDatum> outQueue;
-
- public static final String TAGS_EXTENSION = "tags";
-
- private ElasticsearchWriterConfiguration config;
- private ElasticsearchClientManager manager;
- private BulkRequestBuilder bulkBuilder;
- protected String usePercolateField;
-
- public PercolateTagProcessor(ElasticsearchWriterConfiguration config) {
- this(config, DEFAULT_PERCOLATE_FIELD);
- }
-
- public PercolateTagProcessor(ElasticsearchWriterConfiguration config, String defaultPercolateField) {
- this.config = config;
- this.usePercolateField = defaultPercolateField;
- }
-
- public ElasticsearchClientManager getManager() {
- return manager;
- }
-
- public void setManager(ElasticsearchClientManager manager) {
- this.manager = manager;
- }
-
- public ElasticsearchConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(ElasticsearchWriterConfiguration config) {
- this.config = config;
- }
-
- public Queue<StreamsDatum> getProcessorOutputQueue() {
- return outQueue;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = new ArrayList<>();
-
- String json;
- ObjectNode node;
- // first check for valid json
- if (entry.getDocument() instanceof String) {
- json = (String) entry.getDocument();
- try {
- node = (ObjectNode) mapper.readTree(json);
- } catch (IOException ex) {
- ex.printStackTrace();
- return null;
- }
- } else if (entry.getDocument() instanceof ObjectNode) {
- node = (ObjectNode) entry.getDocument();
- try {
- json = mapper.writeValueAsString(node);
- } catch (JsonProcessingException ex) {
- LOGGER.warn("Invalid datum: ", node);
- return null;
- }
- } else {
- LOGGER.warn("Incompatible document type: ", entry.getDocument().getClass());
- return null;
- }
-
- StringBuilder percolateRequestJson = new StringBuilder();
- percolateRequestJson.append("{ \"doc\": ");
- percolateRequestJson.append(json);
- //percolateRequestJson.append("{ \"content\" : \"crazy good shit\" }");
- percolateRequestJson.append("}");
-
- PercolateRequestBuilder request;
- PercolateResponse response;
-
- try {
- LOGGER.trace("Percolate request json: {}", percolateRequestJson.toString());
- request = manager.client().preparePercolate().setIndices(config.getIndex()).setDocumentType(config.getType()).setSource(percolateRequestJson.toString());
- LOGGER.trace("Percolate request: {}", mapper.writeValueAsString(request.request()));
- response = request.execute().actionGet();
- LOGGER.trace("Percolate response: {} matches", response.getMatches().length);
- } catch (Exception ex) {
- LOGGER.warn("Percolate exception: {}", ex.getMessage());
- return null;
- }
-
- ArrayNode tagArray = JsonNodeFactory.instance.arrayNode();
-
- for (PercolateResponse.Match aResponse : response) {
- tagArray.add(aResponse.getId().string());
- }
-
- LOGGER.trace("Percolate matches: {}", tagArray);
-
- Activity activity = mapper.convertValue(node, Activity.class);
-
- appendMatches(tagArray, activity);
-
- entry.setDocument(activity);
-
- result.add(entry);
-
- return result;
-
- }
-
- protected void appendMatches(ArrayNode tagArray, Activity activity) {
-
- ExtensionUtil.getInstance().addExtension(activity, TAGS_EXTENSION, tagArray);
-
- }
-
- @Override
- public void prepare(Object configuration) {
-
- mapper = StreamsJacksonMapper.getInstance();
-
- Objects.requireNonNull(config);
-
- manager = ElasticsearchClientManager.getInstance(config);
-
- if ( config.getTags() != null && config.getTags().getAdditionalProperties().size() > 0) {
- // initial write tags to index
- createIndexIfMissing(config.getIndex());
- if (config.getReplaceTags()) {
- deleteOldQueries(config.getIndex());
- }
- for (String tag : config.getTags().getAdditionalProperties().keySet()) {
- String query = (String) config.getTags().getAdditionalProperties().get(tag);
- PercolateQueryBuilder queryBuilder = new PercolateQueryBuilder(tag, query, this.usePercolateField);
- addPercolateRule(queryBuilder, config.getIndex());
- }
- bulkBuilder = manager.client().prepareBulk();
-
- if (writePercolateRules()) {
- LOGGER.info("wrote " + bulkBuilder.numberOfActions() + " tags to " + config.getIndex() + " _percolator");
- } else {
- LOGGER.error("FAILED writing " + bulkBuilder.numberOfActions() + " tags to " + config.getIndex() + " _percolator");
- }
- }
-
- }
-
- @Override
- public void cleanUp() {
- if (config.getCleanupTags()) {
- deleteOldQueries(config.getIndex());
- }
- manager.client().close();
- }
-
- public int numOfPercolateRules() {
- return this.bulkBuilder.numberOfActions();
- }
-
- /**
- * createIndexIfMissing.
- * @param indexName indexName
- */
- public void createIndexIfMissing(String indexName) {
- if (!this.manager.client()
- .admin()
- .indices()
- .exists(new IndicesExistsRequest(indexName))
- .actionGet()
- .isExists()) {
- // It does not exist... So we are going to need to create the index.
- // we are going to assume that the 'templates' that we have loaded into
- // elasticsearch are sufficient to ensure the index is being created properly.
- CreateIndexResponse response = this.manager.client().admin().indices().create(new CreateIndexRequest(indexName)).actionGet();
-
- if (response.isAcknowledged()) {
- LOGGER.info("Index {} did not exist. The index was automatically created from the stored ElasticSearch Templates.", indexName);
- } else {
- LOGGER.error("Index {} did not exist. While attempting to create the index from stored ElasticSearch Templates we were unable to get an acknowledgement.", indexName);
- LOGGER.error("Error Message: {}", response.toString());
- throw new RuntimeException("Unable to create index " + indexName);
- }
- }
- }
-
- public void addPercolateRule(PercolateQueryBuilder builder, String index) {
- this.bulkBuilder.add(manager.client().prepareIndex(index, ".percolator", builder.getId())
- .setSource(builder.getSource()));
- }
-
- /**
- *
- * @return returns true if all rules were addded. False indicates one or more rules have failed.
- */
- public boolean writePercolateRules() {
- if (this.numOfPercolateRules() < 0) {
- throw new RuntimeException("No Rules Have been added!");
- }
- BulkResponse response = this.bulkBuilder.execute().actionGet();
- for (BulkItemResponse r : response.getItems()) {
- if (r.isFailed()) {
- LOGGER.error(r.getId() + "\t" + r.getFailureMessage());
- }
- }
- return !response.hasFailures();
- }
-
- /**
- * Attempt to removeOldTags.
- * @param ids ids
- * @param index index
- * @return Returns true if all of the old tags were removed. False indicates one or more tags were not removed.
- */
- public boolean removeOldTags(Set<String> ids, String index) {
- if (ids.size() == 0) {
- return false;
- }
- BulkRequestBuilder bulk = manager.client().prepareBulk();
- for (String id : ids) {
- bulk.add(manager.client().prepareDelete("_percolator", index, id));
- }
- return !bulk.execute().actionGet().hasFailures();
- }
-
- /**
- * get active percolate tags.
- * @param index index
- * @return result
- */
- public Set<String> getActivePercolateTags(String index) {
- Set<String> tags = new HashSet<>();
- SearchRequestBuilder searchBuilder = manager.client().prepareSearch("*").setIndices(index).setTypes(".percolator").setSize(1000);
- SearchResponse response = searchBuilder.setQuery(QueryBuilders.matchAllQuery()).execute().actionGet();
- SearchHits hits = response.getHits();
- for (SearchHit hit : hits.getHits()) {
- tags.add(hit.id());
- }
- return tags;
- }
-
- /**
- * delete old queries.
- * @param index index
- * @return result
- */
- public boolean deleteOldQueries(String index) {
- Set<String> tags = getActivePercolateTags(index);
- if (tags.size() == 0) {
- LOGGER.warn("No active tags were found in _percolator for index : {}", index);
- return false;
- }
- LOGGER.info("Deleting {} tags.", tags.size());
- BulkRequestBuilder bulk = manager.client().prepareBulk();
- for (String tag : tags) {
- bulk.add(manager.client().prepareDelete().setType(".percolator").setIndex(index).setId(tag));
- }
- BulkResponse response = bulk.execute().actionGet();
- return !response.hasFailures();
- }
-
- public static class PercolateQueryBuilder {
-
- private QueryStringQueryBuilder queryBuilder;
- private String id;
-
- /**
- * PercolateQueryBuilder constructor.
- * @param id
- * @param query
- * @param defaultPercolateField
- */
- public PercolateQueryBuilder(String id, String query, String defaultPercolateField) {
- this.id = id;
- this.queryBuilder = new QueryStringQueryBuilder(query);
- this.queryBuilder.defaultField(defaultPercolateField);
- }
-
- public String getId() {
- return this.id;
- }
-
- public String getSource() {
- return "{ \n\"query\" : " + this.queryBuilder.toString() + "\n}";
- }
-
- }
-
- public enum FilterLevel {
- MUST, SHOULD, MUST_NOT
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchConfiguration.json b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchConfiguration.json
deleted file mode 100644
index 37d4fbf..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchConfiguration.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.elasticsearch.ElasticsearchConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "hosts": {
- "type": "array",
- "description": "Elasticsearch Hosts",
- "items": {
- "type": "string"
- }
- },
- "port": {
- "type": "integer",
- "description": "Elasticsearch Transport API port"
- },
- "clusterName": {
- "type": "string",
- "description": "Cluster Name"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchReaderConfiguration.json b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchReaderConfiguration.json
deleted file mode 100644
index 30d1d98..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchReaderConfiguration.json
+++ /dev/null
@@ -1,36 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.elasticsearch.ElasticsearchReaderConfiguration",
- "extends": {"$ref":"ElasticsearchConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "indexes": {
- "type": "array",
- "items": {
- "type": "string"
- },
- "description": "Indexes to read from"
- },
- "types": {
- "type": "array",
- "items": {
- "type": "string"
- },
- "description": "Types to read from"
- },
- "scrollTimeout": {
- "type": "string",
- "description": "Scroll Timeout (JodaTime)"
- },
- "_search": {
- "type": "object",
- "javaType" : "java.util.Map",
- "description": "Search definition"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
deleted file mode 100644
index 79b30c0..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
+++ /dev/null
@@ -1,62 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration",
- "extends": {"$ref":"ElasticsearchConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "index": {
- "type": "string",
- "description": "Index to write to"
- },
- "type": {
- "type": "string",
- "description": "Type to write as"
- },
- "bulk": {
- "type": "boolean",
- "description": "Index in large or small batches"
- },
- "batchSize": {
- "type": "integer",
- "description": "Item Count before flush"
- },
- "batchBytes": {
- "type": "integer",
- "description": "Number of bytes before flush"
- },
- "maxTimeBetweenFlushMs": {
- "type": "integer",
- "description": "Max time between flush (ms)"
- },
- "script": {
- "type": "string",
- "description": "Script to execute during index"
- },
- "tags": {
- "type": "object",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "Tags to apply during index"
- },
- "replaceTags": {
- "type": "boolean",
- "description": "Whether to overwrite _percolator rules on startup"
- },
- "cleanupTags": {
- "type": "boolean",
- "description": "Whether to remove _percolator rules on shutdown"
- },
- "forceUseConfig": {
- "type": "boolean",
- "description": "Whether or not we force the values that are set in the configuration"
- },
- "refresh": {
- "type": "boolean",
- "description": "Whether to refresh during cleanup"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/resources/components.dot b/streams-contrib/streams-persist-elasticsearch/src/main/resources/components.dot
deleted file mode 100644
index f063421..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/resources/components.dot
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
-
- subgraph cluster_processors {
- label="processors";
- processors_elasticsearch_DatumFromMetadataAsDocumentProcessor [label="DatumFromMetadataAsDocumentProcessor"]
- processors_elasticsearch_DatumFromMetadataProcessor [label="DatumFromMetadataProcessor"]
- processors_elasticsearch_DocumentToMetadataProcessor [label="DocumentToMetadataProcessor"]
- processors_elasticsearch_MetadataFromDocumentProcessor [label="MetadataFromDocumentProcessor"]
- processors_elasticsearch_PercolateTagProcessor [label="PercolateTagProcessor"]
- }
-
- subgraph cluster_persisters {
- label="persisters";
- persisters_elasticsearch_reader [label="ElasticsearchPersistReader"]
- persisters_elasticsearch_writer [label="ElasticsearchPersistWriter"]
- persisters_elasticsearch_updater [label="ElasticsearchPersistUpdater"]
- }
-
- subgraph cluster_dbs {
- label="dbs";
- elasticsearch [label="elasticsearch", shape="cylinder"]
- }
-
- generators -> providers
- providers -> processors_elasticsearch_DocumentToMetadataProcessor [lhead=cluster_processors]
- processors_elasticsearch_DocumentToMetadataProcessor -> persisters_elasticsearch_writer [ltail=cluster_processors]
- processors_elasticsearch_DocumentToMetadataProcessor -> persisters_elasticsearch_updater [ltail=cluster_processors]
- persisters_elasticsearch_reader -> processors_elasticsearch_DocumentToMetadataProcessor [label="StreamsDatum[String]", lhead=cluster_processors]
- persisters_elasticsearch_reader <- elasticsearch
- persisters_elasticsearch_writer -> elasticsearch
- persisters_elasticsearch_updater -> elasticsearch
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/resources/elasticsearch-read.conf b/streams-contrib/streams-persist-elasticsearch/src/main/resources/elasticsearch-read.conf
deleted file mode 100644
index eb1d45d..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/resources/elasticsearch-read.conf
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"elasticsearch": {
- "hosts": [
- "localhost"
- ],
- "port": 9300,
- "clusterName": "elasticsearch",
- "indexes": [
- "streams"
- ],
- "types": [
- "post"
- ],
- "_search": {
- "query" : {
- "match_all" : { }
- }
- }
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/resources/elasticsearch-write.conf b/streams-contrib/streams-persist-elasticsearch/src/main/resources/elasticsearch-write.conf
deleted file mode 100644
index 642e60d..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/resources/elasticsearch-write.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"elasticsearch": {
- "hosts": [
- "localhost"
- ],
- "port": 9300,
- "clusterName": "elasticsearch",
- "index": "streams",
- "type": "post"
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/resources/reference.conf b/streams-contrib/streams-persist-elasticsearch/src/main/resources/reference.conf
deleted file mode 100644
index 483ea87..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/resources/reference.conf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-elasticsearch {
- batchSize = 100
- batchBytes = 5242880
- bulk = true
- cleanupTags = true
- maxTimeBetweenFlushMs = 60000
- port = 9300
- refresh = false
- replaceTags = true
- scrollTimeout = 5m
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-elasticsearch/src/site/markdown/elasticsearch.md b/streams-contrib/streams-persist-elasticsearch/src/site/markdown/elasticsearch.md
deleted file mode 100644
index c8a196c..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/site/markdown/elasticsearch.md
+++ /dev/null
@@ -1,35 +0,0 @@
-## Elasticsearch
-
-Start elasticsearch via docker with the docker maven plugin:
-
- mvn -PdockerITs docker:start
-
-Confirm that elasticsearch is running:
-
- docker ps
-
-Confirm that host and post(s) are in property file:
-
- cat elasticsearch.properties
-
-Create a local file `elasticsearch.conf` with cluster details:
-
- elasticsearch {
- hosts += ${es.tcp.host}
- port = ${es.tcp.port}
- clusterName = "elasticsearch"
- }
-
-When configuring a stream, include these files:
-
- include "elasticsearch.properties"
- include "elasticsearch.conf"
-
-Supply application-specific configuration as well:
-
- elasticsearch {
- index: ""
- type: ""
- }
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-elasticsearch/src/site/markdown/index.md b/streams-contrib/streams-persist-elasticsearch/src/site/markdown/index.md
deleted file mode 100644
index f95d5a4..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/site/markdown/index.md
+++ /dev/null
@@ -1,32 +0,0 @@
-streams-persist-elasticsearch
-=====================
-
-Read/write to/from Elasticsearch
-
-## Configuration
-
-| Schema |
-|--------|
-| [ElasticsearchConfiguration.json](org/apache/streams/elasticsearch/ElasticsearchConfiguration.json "ElasticsearchConfiguration.json") [ElasticsearchConfiguration.html](apidocs/org/apache/streams/elasticsearch/ElasticsearchConfiguration.html "javadoc") |
-
-## Components
-
-
-
-| Class | Configuration | Example Configuration(s) |
-|-------|---------------|--------------------------|
-| ElasticsearchPersistReader [ElasticsearchPersistReader.html](apidocs/org/apache/streams/elasticsearch/ElasticsearchPersistReader.html "javadoc") | [ElasticsearchReaderConfiguration.json](org/apache/streams/elasticsearch/ElasticsearchReaderConfiguration.json "ElasticsearchReaderConfiguration.json") [ElasticsearchReaderConfiguration.html](apidocs/org/apache/streams/elasticsearch/ElasticsearchReaderConfiguration.html "javadoc") | [elasticsearch-read.conf](elasticsearch-read.conf "elastics [...]
-| ElasticsearchPersistWriter [ElasticsearchPersistWriter.html](apidocs/org/apache/streams/elasticsearch/ElasticsearchPersistWriter "javadoc") | [ElasticsearchWriterConfiguration.json](org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json "ElasticsearchWriterConfiguration.json") [ElasticsearchWriterConfiguration.html](apidocs/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.html "javadoc") | [elasticsearch-write.conf](elasticsearch-write.conf "elasticsear [...]
-| ElasticsearchPersistUpdater [ElasticsearchPersistUpdater.html](apidocs/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater "javadoc") | [ElasticsearchWriterConfiguration.json](org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json "ElasticsearchWriterConfiguration.json") [ElasticsearchWriterConfiguration.html](apidocs/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.html "javadoc") | [elasticsearch-write.conf](elasticsearch-write.conf "elastics [...]
-
-Testing:
----------
-
- mvn -PdockerITs docker:start
- mvn clean install test verify -DskipITs=false
- mvn -PdockerITs docker:stop
-
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-elasticsearch/src/site/site.xml b/streams-contrib/streams-persist-elasticsearch/src/site/site.xml
deleted file mode 100644
index e4a3647..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/site/site.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project>
- <body>
- <links name="Help">
- <item name="Elasticsearch" href="elasticsearch.html"/>
- </links>
- </body>
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/processor/PercolateTagProcessorTest.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/processor/PercolateTagProcessorTest.java
deleted file mode 100644
index e252901..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/processor/PercolateTagProcessorTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.processor;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Unit Test for
- * @see org.apache.streams.elasticsearch.processor.PercolateTagProcessor
- */
-public class PercolateTagProcessorTest {
-
- private final String id = "test_id";
- private final String query = "test_query";
- private final String defaultPercolateField = "activity.content";
-
- private final String expectedResults = "{ \n" +
- "\"query\" : {\n" +
- " \"query_string\" : {\n" +
- " \"query\" : \"test_query\",\n" +
- " \"default_field\" : \"activity.content\"\n" +
- " }\n" +
- "}\n" +
- "}";
-
- @Test
- public void percolateTagProcessorQueryBuilderTest() {
- PercolateTagProcessor.PercolateQueryBuilder percolateQueryBuilder = new PercolateTagProcessor.PercolateQueryBuilder(id, query, defaultPercolateField);
-
- assertEquals(id, percolateQueryBuilder.getId());
-// assertEquals(expectedResults, percolateQueryBuilder.getSource());
- }
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/DatumFromMetadataProcessorIT.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/DatumFromMetadataProcessorIT.java
deleted file mode 100644
index d361436..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/DatumFromMetadataProcessorIT.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.test;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.elasticsearch.ElasticsearchClientManager;
-import org.apache.streams.elasticsearch.ElasticsearchReaderConfiguration;
-import org.apache.streams.elasticsearch.processor.DatumFromMetadataProcessor;
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.commons.lang.SerializationUtils;
-import org.elasticsearch.client.Client;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.testng.Assert.assertTrue;
-
-/**
- * Integration Test for
- * @see org.apache.streams.elasticsearch.processor.DatumFromMetadataProcessor
- */
-@Test( groups={"DatumFromMetadataProcessorIT"},
- dependsOnGroups={"ElasticsearchPersistWriterIT"}
- )
-public class DatumFromMetadataProcessorIT {
-
- private ElasticsearchReaderConfiguration testConfiguration;
- protected Client testClient;
-
- @Test
- public void testSerializability() {
- DatumFromMetadataProcessor processor = new DatumFromMetadataProcessor(testConfiguration);
-
- DatumFromMetadataProcessor clone = (DatumFromMetadataProcessor) SerializationUtils.clone(processor);
- }
-
- @BeforeClass
- public void prepareTestDatumFromMetadataProcessor() throws Exception {
-
- Config reference = ConfigFactory.load();
- File conf_file = new File("target/test-classes/DatumFromMetadataProcessorIT.conf");
- assertTrue(conf_file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(ElasticsearchReaderConfiguration.class).detectConfiguration(typesafe, "elasticsearch");
- testClient = ElasticsearchClientManager.getInstance(testConfiguration).client();
-
- }
-
- @Test()
- public void testDatumFromMetadataProcessor() {
-
- Map<String, Object> metadata = new HashMap<>();
-
- metadata.put("index", testConfiguration.getIndexes().get(0));
- metadata.put("type", testConfiguration.getTypes().get(0));
- metadata.put("id", "post");
-
- DatumFromMetadataProcessor processor = new DatumFromMetadataProcessor(testConfiguration);
-
- StreamsDatum testInput = new StreamsDatum(null);
-
- testInput.setMetadata(metadata);
-
- Assert.assertNull(testInput.document);
-
- processor.prepare(null);
-
- StreamsDatum testOutput = processor.process(testInput).get(0);
-
- processor.cleanUp();
-
- Assert.assertNotNull(testOutput.document);
-
- }
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildUpdaterIT.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildUpdaterIT.java
deleted file mode 100644
index 97a88d4..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildUpdaterIT.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.test;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.elasticsearch.ElasticsearchClientManager;
-import org.apache.streams.elasticsearch.ElasticsearchPersistUpdater;
-import org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.reflections.Reflections;
-import org.reflections.scanners.SubTypesScanner;
-import org.reflections.util.ClasspathHelper;
-import org.reflections.util.ConfigurationBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.Set;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotEquals;
-import static org.testng.Assert.assertTrue;
-
-/**
- * Integration Test for
- * @see org.apache.streams.elasticsearch.ElasticsearchPersistUpdater
- * using parent/child associated documents.
- */
-@Test
- (
- groups={"ElasticsearchParentChildUpdaterIT"},
- dependsOnGroups={"ElasticsearchParentChildWriterIT"}
- )
-public class ElasticsearchParentChildUpdaterIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchParentChildUpdaterIT.class);
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- protected ElasticsearchWriterConfiguration testConfiguration;
- protected Client testClient;
-
- Set<Class<? extends ActivityObject>> objectTypes;
-
- List<String> files;
-
- @BeforeClass
- public void prepareTestParentChildPersistUpdater() throws Exception {
-
- Config reference = ConfigFactory.load();
- File conf_file = new File("target/test-classes/ElasticsearchParentChildUpdaterIT.conf");
- assertTrue(conf_file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(ElasticsearchWriterConfiguration.class).detectConfiguration(typesafe, "elasticsearch");
- testClient = ElasticsearchClientManager.getInstance(testConfiguration).client();
-
- ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest();
- ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest).actionGet();
- assertNotEquals(clusterHealthResponse.getStatus(), ClusterHealthStatus.RED);
-
- IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getIndex());
- IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet();
- assertTrue(indicesExistsResponse.isExists());
-
- Reflections reflections = new Reflections(new ConfigurationBuilder()
- .setUrls(ClasspathHelper.forPackage("org.apache.streams.pojo.json"))
- .setScanners(new SubTypesScanner()));
- objectTypes = reflections.getSubTypesOf(ActivityObject.class);
-
- InputStream testActivityFolderStream = ElasticsearchParentChildUpdaterIT.class.getClassLoader()
- .getResourceAsStream("activities");
- files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- }
-
- @Test
- public void testParentChildPersistUpdater() throws Exception {
-
- ElasticsearchPersistUpdater testPersistUpdater = new ElasticsearchPersistUpdater(testConfiguration);
- testPersistUpdater.prepare(null);
-
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = ElasticsearchParentChildUpdaterIT.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- activity.setAdditionalProperty("updated", Boolean.TRUE);
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- if( !StringUtils.isEmpty(activity.getObject().getObjectType())) {
- datum.getMetadata().put("parent", activity.getObject().getObjectType());
- datum.getMetadata().put("type", "activity");
- testPersistUpdater.write(datum);
- LOGGER.info("Updated: " + activity.getVerb() );
- }
- }
-
- testPersistUpdater.cleanUp();
-
- SearchRequestBuilder countUpdatedRequest = testClient
- .prepareSearch(testConfiguration.getIndex())
- .setTypes("activity")
- .setQuery(QueryBuilders.queryStringQuery("updated:true"));
- SearchResponse countUpdatedResponse = countUpdatedRequest.execute().actionGet();
-
- assertEquals(84, countUpdatedResponse.getHits().getTotalHits());
-
- }
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildWriterIT.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildWriterIT.java
deleted file mode 100644
index 31c7f11..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildWriterIT.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.test;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.elasticsearch.ElasticsearchClientManager;
-import org.apache.streams.elasticsearch.ElasticsearchPersistWriter;
-import org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
-
-import org.reflections.Reflections;
-import org.reflections.scanners.SubTypesScanner;
-import org.reflections.util.ClasspathHelper;
-import org.reflections.util.ConfigurationBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.InputStream;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.Set;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotEquals;
-import static org.testng.Assert.assertTrue;
-
-/**
- * Integration Test for
- * @see org.apache.streams.elasticsearch.ElasticsearchPersistWriter
- * using parent/child associated documents.
- */
-@Test
- (
- groups={"ElasticsearchParentChildWriterIT"}
- )
-public class ElasticsearchParentChildWriterIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchParentChildWriterIT.class);
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- protected ElasticsearchWriterConfiguration testConfiguration;
- protected Client testClient;
-
- Set<Class<? extends ActivityObject>> objectTypes;
-
- List<String> files;
-
- @BeforeClass
- public void prepareTestParentChildPersistWriter() throws Exception {
-
- Config reference = ConfigFactory.load();
- File conf_file = new File("target/test-classes/ElasticsearchParentChildWriterIT.conf");
- assertTrue(conf_file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(ElasticsearchWriterConfiguration.class).detectConfiguration(typesafe, "elasticsearch");
- testClient = ElasticsearchClientManager.getInstance(testConfiguration).client();
-
- ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest();
- ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest).actionGet();
- assertNotEquals(clusterHealthResponse.getStatus(), ClusterHealthStatus.RED);
-
- IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getIndex());
- IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet();
- if(indicesExistsResponse.isExists()) {
- DeleteIndexRequest deleteIndexRequest = Requests.deleteIndexRequest(testConfiguration.getIndex());
- DeleteIndexResponse deleteIndexResponse = testClient.admin().indices().delete(deleteIndexRequest).actionGet();
- assertTrue(deleteIndexResponse.isAcknowledged());
- }
-
- PutIndexTemplateRequestBuilder putTemplateRequestBuilder = testClient.admin().indices().preparePutTemplate("mappings");
- URL templateURL = ElasticsearchParentChildWriterIT.class.getResource("/ActivityChildObjectParent.json");
- ObjectNode template = MAPPER.readValue(templateURL, ObjectNode.class);
- String templateSource = MAPPER.writeValueAsString(template);
- putTemplateRequestBuilder.setSource(templateSource);
-
- testClient.admin().indices().putTemplate(putTemplateRequestBuilder.request()).actionGet();
-
- Reflections reflections = new Reflections(new ConfigurationBuilder()
- .setUrls(ClasspathHelper.forPackage("org.apache.streams.pojo.json"))
- .setScanners(new SubTypesScanner()));
- objectTypes = reflections.getSubTypesOf(ActivityObject.class);
-
- InputStream testActivityFolderStream = ElasticsearchParentChildWriterIT.class.getClassLoader()
- .getResourceAsStream("activities");
- files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- }
-
- @Test
- public void testParentChildPersistWriter() throws Exception {
-
- IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getIndex());
- IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet();
- if(indicesExistsResponse.isExists()) {
- DeleteIndexRequest deleteIndexRequest = Requests.deleteIndexRequest(testConfiguration.getIndex());
- DeleteIndexResponse deleteIndexResponse = testClient.admin().indices().delete(deleteIndexRequest).actionGet();
- }
-
- ElasticsearchPersistWriter testPersistWriter = new ElasticsearchPersistWriter(testConfiguration);
- testPersistWriter.prepare(null);
-
- for( Class objectType : objectTypes ) {
- Object object = objectType.newInstance();
- ActivityObject activityObject = MAPPER.convertValue(object, ActivityObject.class);
- StreamsDatum datum = new StreamsDatum(activityObject, activityObject.getObjectType());
- datum.getMetadata().put("type", "object");
- testPersistWriter.write( datum );
- }
-
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = ElasticsearchParentChildWriterIT.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- if( !StringUtils.isEmpty(activity.getObject().getObjectType())) {
- datum.getMetadata().put("parent", activity.getObject().getObjectType());
- datum.getMetadata().put("type", "activity");
- testPersistWriter.write(datum);
- LOGGER.info("Wrote: " + activity.getVerb());
- }
- }
-
- testPersistWriter.cleanUp();
-
- SearchRequestBuilder countParentRequest = testClient
- .prepareSearch(testConfiguration.getIndex())
- .setTypes("object");
- SearchResponse countParentResponse = countParentRequest.execute().actionGet();
-
- assertEquals(41, countParentResponse.getHits().getTotalHits());
-
- SearchRequestBuilder countChildRequest = testClient
- .prepareSearch(testConfiguration.getIndex())
- .setTypes("activity");
- SearchResponse countChildResponse = countChildRequest.execute().actionGet();
-
- assertEquals(84, countChildResponse.getHits().getTotalHits());
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistUpdaterIT.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistUpdaterIT.java
deleted file mode 100644
index 7eded15..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistUpdaterIT.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.test;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.elasticsearch.ElasticsearchClientManager;
-import org.apache.streams.elasticsearch.ElasticsearchPersistUpdater;
-import org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-
-import org.apache.commons.io.IOUtils;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotEquals;
-import static org.testng.Assert.assertTrue;
-
-/**
- * Integration Test for
- * @see org.apache.streams.elasticsearch.ElasticsearchPersistUpdater
- */
-@Test
- (
- groups={"ElasticsearchPersistUpdaterIT"},
- dependsOnGroups={"ElasticsearchPersistWriterIT"}
- )
-public class ElasticsearchPersistUpdaterIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistUpdaterIT.class);
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- protected ElasticsearchWriterConfiguration testConfiguration;
- protected Client testClient;
-
- @BeforeClass
- public void prepareTestPersistUpdater() throws Exception {
-
- Config reference = ConfigFactory.load();
- File conf_file = new File("target/test-classes/ElasticsearchPersistUpdaterIT.conf");
- assertTrue(conf_file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(ElasticsearchWriterConfiguration.class).detectConfiguration(typesafe, "elasticsearch");
- testClient = ElasticsearchClientManager.getInstance(testConfiguration).client();
-
- ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest();
- ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest).actionGet();
- assertNotEquals(clusterHealthResponse.getStatus(), ClusterHealthStatus.RED);
-
- IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getIndex());
- IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet();
- assertTrue(indicesExistsResponse.isExists());
-
- }
-
- @Test
- public void testPersistUpdater() throws Exception {
-
- IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getIndex());
- IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet();
- assertTrue(indicesExistsResponse.isExists());
-
- SearchRequestBuilder countRequest = testClient
- .prepareSearch(testConfiguration.getIndex())
- .setTypes(testConfiguration.getType());
- SearchResponse countResponse = countRequest.execute().actionGet();
-
- long count = countResponse.getHits().getTotalHits();
-
- ElasticsearchPersistUpdater testPersistUpdater = new ElasticsearchPersistUpdater(testConfiguration);
- testPersistUpdater.prepare(null);
-
- InputStream testActivityFolderStream = ElasticsearchPersistUpdaterIT.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = ElasticsearchPersistUpdaterIT.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- Activity update = new Activity();
- update.setAdditionalProperty("updated", Boolean.TRUE);
- update.setAdditionalProperty("str", "str");
- update.setAdditionalProperty("long", 10L);
- update.setActor(
- new ActivityObject()
- .withAdditionalProperty("updated", Boolean.TRUE)
- .withAdditionalProperty("double", 10d)
- .withAdditionalProperty("map",
- MAPPER.createObjectNode().set("field", MAPPER.createArrayNode().add("item"))));
-
- StreamsDatum datum = new StreamsDatum(update, activity.getVerb());
- testPersistUpdater.write( datum );
- LOGGER.info("Updated: " + activity.getVerb() );
- }
-
- testPersistUpdater.cleanUp();
-
- SearchRequestBuilder updatedCountRequest = testClient
- .prepareSearch(testConfiguration.getIndex())
- .setTypes(testConfiguration.getType())
- .setQuery(QueryBuilders.existsQuery("updated"));
- SearchResponse updatedCount = updatedCountRequest.execute().actionGet();
-
- LOGGER.info("updated: {}", updatedCount.getHits().getTotalHits());
-
- assertEquals(count, updatedCount.getHits().getTotalHits());
-
- SearchRequestBuilder actorUpdatedCountRequest = testClient
- .prepareSearch(testConfiguration.getIndex())
- .setTypes(testConfiguration.getType())
- .setQuery(QueryBuilders.termQuery("actor.updated", true));
- SearchResponse actorUpdatedCount = actorUpdatedCountRequest.execute().actionGet();
-
- LOGGER.info("actor.updated: {}", actorUpdatedCount.getHits().getTotalHits());
-
- assertEquals(count, actorUpdatedCount.getHits().getTotalHits());
-
- SearchRequestBuilder strUpdatedCountRequest = testClient
- .prepareSearch(testConfiguration.getIndex())
- .setTypes(testConfiguration.getType())
- .setQuery(QueryBuilders.termQuery("str", "str"));
- SearchResponse strUpdatedCount = strUpdatedCountRequest.execute().actionGet();
-
- LOGGER.info("strupdated: {}", strUpdatedCount.getHits().getTotalHits());
-
- assertEquals(count, strUpdatedCount.getHits().getTotalHits());
-
- SearchRequestBuilder longUpdatedCountRequest = testClient
- .prepareSearch(testConfiguration.getIndex())
- .setTypes(testConfiguration.getType())
- .setQuery(QueryBuilders.rangeQuery("long").from(9).to(11));
- SearchResponse longUpdatedCount = longUpdatedCountRequest.execute().actionGet();
-
- LOGGER.info("longupdated: {}", longUpdatedCount.getHits().getTotalHits());
-
- assertEquals(count, longUpdatedCount.getHits().getTotalHits());
-
- SearchRequestBuilder doubleUpdatedCountRequest = testClient
- .prepareSearch(testConfiguration.getIndex())
- .setTypes(testConfiguration.getType())
- .setQuery(QueryBuilders.rangeQuery("long").from(9).to(11));
- SearchResponse doubleUpdatedCount = doubleUpdatedCountRequest.execute().actionGet();
-
- LOGGER.info("doubleupdated: {}", doubleUpdatedCount.getHits().getTotalHits());
-
- assertEquals(count, doubleUpdatedCount.getHits().getTotalHits());
-
- SearchRequestBuilder mapUpdatedCountRequest = testClient
- .prepareSearch(testConfiguration.getIndex())
- .setTypes(testConfiguration.getType())
- .setQuery(QueryBuilders.termQuery("actor.map.field", "item"));
- SearchResponse mapUpdatedCount = mapUpdatedCountRequest.execute().actionGet();
-
- LOGGER.info("mapfieldupdated: {}", mapUpdatedCount.getHits().getTotalHits());
-
- assertEquals(count, mapUpdatedCount.getHits().getTotalHits());
-
- }
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistWriterIT.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistWriterIT.java
deleted file mode 100644
index caa719e..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistWriterIT.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.test;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.elasticsearch.ElasticsearchClientManager;
-import org.apache.streams.elasticsearch.ElasticsearchPersistWriter;
-import org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.commons.io.IOUtils;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotEquals;
-import static org.testng.Assert.assertTrue;
-
-/**
- * Integration Test for
- * @see org.apache.streams.elasticsearch.ElasticsearchPersistWriter
- */
-@Test
- (
- groups={"ElasticsearchPersistWriterIT"}
- )
-public class ElasticsearchPersistWriterIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistWriterIT.class);
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private ElasticsearchWriterConfiguration testConfiguration;
- private Client testClient;
-
- @BeforeClass
- public void prepareTestPersistWriter() throws Exception {
-
- Config reference = ConfigFactory.load();
- File conf_file = new File("target/test-classes/ElasticsearchPersistWriterIT.conf");
- assertTrue(conf_file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(ElasticsearchWriterConfiguration.class).detectConfiguration(typesafe, "elasticsearch");
- testClient = ElasticsearchClientManager.getInstance(testConfiguration).client();
-
- ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest();
- ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest).actionGet();
- assertNotEquals(clusterHealthResponse.getStatus(), ClusterHealthStatus.RED);
-
- IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getIndex());
- IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet();
- if(indicesExistsResponse.isExists()) {
- DeleteIndexRequest deleteIndexRequest = Requests.deleteIndexRequest(testConfiguration.getIndex());
- DeleteIndexResponse deleteIndexResponse = testClient.admin().indices().delete(deleteIndexRequest).actionGet();
- assertTrue(deleteIndexResponse.isAcknowledged());
- }
-
- }
-
- @Test
- public void testPersistWriter() throws Exception {
-
- ElasticsearchPersistWriter testPersistWriter = new ElasticsearchPersistWriter(testConfiguration);
- testPersistWriter.prepare(null);
-
- InputStream testActivityFolderStream = ElasticsearchPersistWriterIT.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = ElasticsearchPersistWriterIT.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- testPersistWriter.write( datum );
- LOGGER.info("Wrote: " + activity.getVerb() );
- }
-
- testPersistWriter.cleanUp();
-
- SearchRequestBuilder countRequest = testClient
- .prepareSearch(testConfiguration.getIndex())
- .setTypes(testConfiguration.getType());
- SearchResponse countResponse = countRequest.execute().actionGet();
-
- assertEquals(89, countResponse.getHits().getTotalHits());
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/TestDocumentToMetadataProcessor.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/TestDocumentToMetadataProcessor.java
deleted file mode 100644
index cd40b9d..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/TestDocumentToMetadataProcessor.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.test;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.elasticsearch.processor.DocumentToMetadataProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import org.apache.commons.lang.SerializationUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestDocumentToMetadataProcessor {
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- @Before
- public void prepareTest() {
-
- }
-
- @Test
- public void testSerializability() {
- DocumentToMetadataProcessor processor = new DocumentToMetadataProcessor();
-
- DocumentToMetadataProcessor clone = (DocumentToMetadataProcessor) SerializationUtils.clone(processor);
- }
-
- @Test
- public void testDocumentToMetadataProcessor() {
-
- ObjectNode document = MAPPER.createObjectNode()
- .put("a", "a")
- .put("b", "b")
- .put("c", 6);
-
- DocumentToMetadataProcessor processor = new DocumentToMetadataProcessor();
-
- StreamsDatum testInput = new StreamsDatum(document);
-
- Assert.assertNotNull(testInput.document);
- Assert.assertNotNull(testInput.metadata);
- Assert.assertEquals(testInput.metadata.size(), 0);
-
- processor.prepare(null);
-
- StreamsDatum testOutput = processor.process(testInput).get(0);
-
- processor.cleanUp();
-
- Assert.assertNotNull(testOutput.metadata);
- Assert.assertEquals(testInput.metadata.size(), 3);
-
- }
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/TestMetadataFromDocumentProcessor.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/TestMetadataFromDocumentProcessor.java
deleted file mode 100644
index a0b483f..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/TestMetadataFromDocumentProcessor.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.elasticsearch.test;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.elasticsearch.processor.MetadataFromDocumentProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.SerializationUtils;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * Unit Test for
- * @see org.apache.streams.elasticsearch.processor.MetadataFromDocumentProcessor
- */
-public class TestMetadataFromDocumentProcessor {
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TestMetadataFromDocumentProcessor.class);
-
- @Before
- public void prepareTest() {
-
- }
-
- @Test
- public void testSerializability() {
- MetadataFromDocumentProcessor processor = new MetadataFromDocumentProcessor();
-
- MetadataFromDocumentProcessor clone = (MetadataFromDocumentProcessor) SerializationUtils.clone(processor);
- }
-
- @Test
- public void testMetadataFromDocumentProcessor() throws Exception {
-
- MetadataFromDocumentProcessor processor = new MetadataFromDocumentProcessor();
-
- processor.prepare(null);
-
- InputStream testActivityFolderStream = TestMetadataFromDocumentProcessor.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- Set<ActivityObject> objects = new HashSet<>();
-
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = TestMetadataFromDocumentProcessor.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- activity.setId(activity.getVerb());
- activity.getAdditionalProperties().remove("$license");
-
- if( activity.getActor().getObjectType() != null)
- objects.add(activity.getActor());
- if( activity.getObject().getObjectType() != null)
- objects.add(activity.getObject());
-
- StreamsDatum datum = new StreamsDatum(activity);
-
- List<StreamsDatum> resultList = processor.process(datum);
- assertNotNull(resultList);
- assertEquals(1, resultList.size());
-
- StreamsDatum result = resultList.get(0);
- assertNotNull(result);
- assertNotNull(result.getDocument());
- assertNotNull(result.getId());
- assertNotNull(result.getMetadata());
- assertNotNull(result.getMetadata().get("id"));
- assertNotNull(result.getMetadata().get("type"));
-
- LOGGER.info("valid: " + activity.getVerb() );
- }
-
- for( ActivityObject activityObject : objects) {
- LOGGER.info("Object: " + MAPPER.writeValueAsString(activityObject));
-
- activityObject.setId(activityObject.getObjectType());
- StreamsDatum datum = new StreamsDatum(activityObject);
-
- List<StreamsDatum> resultList = processor.process(datum);
- assertNotNull(resultList);
- assertEquals(1, resultList.size());
-
- StreamsDatum result = resultList.get(0);
- assertNotNull(result);
- assertNotNull(result.getDocument());
- assertNotNull(result.getId());
- assertNotNull(result.getMetadata());
- assertNotNull(result.getMetadata().get("id"));
- assertNotNull(result.getMetadata().get("type"));
-
- LOGGER.info("valid: " + activityObject.getObjectType() );
- }
- }
-}
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/resources/ActivityChildObjectParent.json b/streams-contrib/streams-persist-elasticsearch/src/test/resources/ActivityChildObjectParent.json
deleted file mode 100644
index 22c1c15..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/resources/ActivityChildObjectParent.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "template": "*_parent_child_it",
- "order": 100,
- "mappings": {
- "object": {
- "dynamic": true
- },
- "activity": {
- "_parent": {
- "type": "object"
- },
- "_routing": {
- "required": true
- },
- "dynamic": true
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/resources/DatumFromMetadataProcessorIT.conf b/streams-contrib/streams-persist-elasticsearch/src/test/resources/DatumFromMetadataProcessorIT.conf
deleted file mode 100644
index 7031388..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/resources/DatumFromMetadataProcessorIT.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-elasticsearch {
- hosts += ${es.tcp.host}
- port = ${es.tcp.port}
- clusterName = "elasticsearch"
- indexes += "elasticsearch_persist_writer_it"
- types += "activity"
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/resources/ElasticsearchParentChildUpdaterIT.conf b/streams-contrib/streams-persist-elasticsearch/src/test/resources/ElasticsearchParentChildUpdaterIT.conf
deleted file mode 100644
index 2d95887..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/resources/ElasticsearchParentChildUpdaterIT.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-elasticsearch {
- hosts += ${es.tcp.host}
- port = ${es.tcp.port}
- clusterName = "elasticsearch"
- index = "elasticsearch_persist_writer_parent_child_it"
- batchSize = 5
- refresh = true
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/resources/ElasticsearchParentChildWriterIT.conf b/streams-contrib/streams-persist-elasticsearch/src/test/resources/ElasticsearchParentChildWriterIT.conf
deleted file mode 100644
index 2d95887..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/resources/ElasticsearchParentChildWriterIT.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-elasticsearch {
- hosts += ${es.tcp.host}
- port = ${es.tcp.port}
- clusterName = "elasticsearch"
- index = "elasticsearch_persist_writer_parent_child_it"
- batchSize = 5
- refresh = true
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/resources/ElasticsearchPersistUpdaterIT.conf b/streams-contrib/streams-persist-elasticsearch/src/test/resources/ElasticsearchPersistUpdaterIT.conf
deleted file mode 100644
index 26fc418..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/resources/ElasticsearchPersistUpdaterIT.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-elasticsearch {
- hosts += ${es.tcp.host}
- port = ${es.tcp.port}
- clusterName = "elasticsearch"
- index = "elasticsearch_persist_writer_it"
- type = "activity"
- refresh = true
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/resources/ElasticsearchPersistWriterIT.conf b/streams-contrib/streams-persist-elasticsearch/src/test/resources/ElasticsearchPersistWriterIT.conf
deleted file mode 100644
index 26fc418..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/test/resources/ElasticsearchPersistWriterIT.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-elasticsearch {
- hosts += ${es.tcp.host}
- port = ${es.tcp.port}
- clusterName = "elasticsearch"
- index = "elasticsearch_persist_writer_it"
- type = "activity"
- refresh = true
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-filebuffer/README.md b/streams-contrib/streams-persist-filebuffer/README.md
deleted file mode 100644
index f726141..0000000
--- a/streams-contrib/streams-persist-filebuffer/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-persist-filebuffer
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-persist-filebuffer/pom.xml b/streams-contrib/streams-persist-filebuffer/pom.xml
deleted file mode 100644
index b92be1e..0000000
--- a/streams-contrib/streams-persist-filebuffer/pom.xml
+++ /dev/null
@@ -1,141 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-filebuffer</artifactId>
-
- <properties>
- <tape.version>1.2.3</tape.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.squareup</groupId>
- <artifactId>tape</artifactId>
- <version>${tape.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-runtime-local</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-persist-console</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- </dependency>
- </dependencies>
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.file.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-filebuffer/src/main/java/org/apache/streams/filebuffer/FileBufferPersistReader.java b/streams-contrib/streams-persist-filebuffer/src/main/java/org/apache/streams/filebuffer/FileBufferPersistReader.java
deleted file mode 100644
index 32d6032..0000000
--- a/streams-contrib/streams-persist-filebuffer/src/main/java/org/apache/streams/filebuffer/FileBufferPersistReader.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.filebuffer;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.squareup.tape.QueueFile;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-/**
- * Reads data from a buffer stored on the file-system.
- */
-public class FileBufferPersistReader implements StreamsPersistReader, Serializable {
-
- private static final String STREAMS_ID = "FileBufferPersistReader";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FileBufferPersistReader.class);
-
- private volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper;
-
- private FileBufferConfiguration config;
-
- private QueueFile queueFile;
-
- private boolean isStarted = false;
- private boolean isStopped = false;
-
- private ExecutorService executor = Executors.newSingleThreadExecutor();
-
- public FileBufferPersistReader() {
- this(new ComponentConfigurator<>(FileBufferConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("filebuffer")));
- }
-
- public FileBufferPersistReader(FileBufferConfiguration config) {
- this.config = config;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public StreamsResultSet readAll() {
- return readCurrent();
- }
-
- @Override
- public void startStream() {
- isStarted = true;
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- while (!queueFile.isEmpty()) {
- try {
- byte[] bytes = queueFile.peek();
- ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
- BufferedReader buf = new BufferedReader(new InputStreamReader(bais));
- String line = buf.readLine();
- LOGGER.debug(line);
- write(new StreamsDatum(line));
- queueFile.remove();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- StreamsResultSet current;
- current = new StreamsResultSet(new ConcurrentLinkedQueue<>(persistQueue));
- persistQueue.clear();
-
- return current;
- }
-
- private void write( StreamsDatum entry ) {
- persistQueue.offer(entry);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger bigInteger) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime dateTime, DateTime dateTime2) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return isStarted && !isStopped;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- try {
- Thread.sleep(1000);
- } catch (InterruptedException ie) {
- //Handle exception
- }
-
- mapper = new ObjectMapper();
-
- File file = new File( config.getPath());
-
- if ( !file.exists() ) {
- try {
- file.createNewFile();
- } catch (IOException ex) {
- LOGGER.error(ex.getMessage());
- }
- }
-
- Preconditions.checkArgument(file.exists());
- Preconditions.checkArgument(file.canRead());
-
- try {
- queueFile = new QueueFile(file);
- } catch (IOException ex) {
- LOGGER.error(ex.getMessage());
- }
-
- Objects.requireNonNull(queueFile);
-
- this.persistQueue = new ConcurrentLinkedQueue<>();
-
- }
-
- @Override
- public void cleanUp() {
- try {
- queueFile.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- } finally {
- queueFile = null;
- isStopped = true;
- }
- }
-}
diff --git a/streams-contrib/streams-persist-filebuffer/src/main/java/org/apache/streams/filebuffer/FileBufferPersistWriter.java b/streams-contrib/streams-persist-filebuffer/src/main/java/org/apache/streams/filebuffer/FileBufferPersistWriter.java
deleted file mode 100644
index a4348f6..0000000
--- a/streams-contrib/streams-persist-filebuffer/src/main/java/org/apache/streams/filebuffer/FileBufferPersistWriter.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.filebuffer;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.util.GuidUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.squareup.tape.QueueFile;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-/**
- * Writes data to a buffer stored on the file-system.
- */
-public class FileBufferPersistWriter implements StreamsPersistWriter, Serializable {
-
- private static final String STREAMS_ID = "FileBufferPersistWriter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FileBufferPersistWriter.class);
-
- private volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper;
-
- private FileBufferConfiguration config;
-
- private QueueFile queueFile;
-
- public FileBufferPersistWriter() {
- this(new ComponentConfigurator<>(FileBufferConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("filebuffer")));
- }
-
- public FileBufferPersistWriter(FileBufferConfiguration config) {
- this.config = config;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void write(StreamsDatum entry) {
-
- String key = entry.getId() != null ? entry.getId() : GuidUtils.generateGuid("filewriter");
-
- Preconditions.checkArgument(StringUtils.isNotBlank(key));
- Preconditions.checkArgument(entry.getDocument() instanceof String);
- Preconditions.checkArgument(StringUtils.isNotBlank((String) entry.getDocument()));
-
- byte[] item = ((String)entry.getDocument()).getBytes();
- try {
- queueFile.add(item);
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- mapper = new ObjectMapper();
-
- File file = new File( config.getPath());
-
- try {
- queueFile = new QueueFile(file);
- } catch (IOException ex) {
- ex.printStackTrace();
- }
-
- Preconditions.checkArgument(file.exists());
- Preconditions.checkArgument(file.canWrite());
-
- Objects.requireNonNull(queueFile);
-
- this.persistQueue = new ConcurrentLinkedQueue<>();
-
- }
-
- @Override
- public void cleanUp() {
- try {
- queueFile.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- } finally {
- queueFile = null;
- }
- }
-}
diff --git a/streams-contrib/streams-persist-filebuffer/src/main/jsonschema/org/apache/streams/file/FileBufferConfiguration.json b/streams-contrib/streams-persist-filebuffer/src/main/jsonschema/org/apache/streams/file/FileBufferConfiguration.json
deleted file mode 100644
index 2d9fa5f..0000000
--- a/streams-contrib/streams-persist-filebuffer/src/main/jsonschema/org/apache/streams/file/FileBufferConfiguration.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.filebuffer.FileBufferConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "path": {
- "type": "string",
- "description": "A file to read/write data",
- "default": "/tmp/streams-file-queue.txt"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-filebuffer/src/main/resources/components.dot b/streams-contrib/streams-persist-filebuffer/src/main/resources/components.dot
deleted file mode 100644
index 3ecd8cb..0000000
--- a/streams-contrib/streams-persist-filebuffer/src/main/resources/components.dot
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- processors [label="processors", shape="circle"];
-
- subgraph cluster_persisters {
- label="persisters";
- persisters_filebuffer_reader [label="FileBufferPersistReader"]
- persisters_filebuffer_writer [label="FileBufferPersistWriter"]
- }
-
- subgraph cluster_queues {
- label="queues";
- filebuffer [label="filebuffer"]
- }
-
- generators -> providers
- providers -> processors
- processors -> persisters_filebuffer_writer [label="StreamsDatum"]
- persisters_filebuffer_reader -> processors [label="StreamsDatum[String]"]
- persisters_filebuffer_writer -> filebuffer
- filebuffer -> persisters_filebuffer_reader
-}
diff --git a/streams-contrib/streams-persist-filebuffer/src/main/resources/filebuffer.conf b/streams-contrib/streams-persist-filebuffer/src/main/resources/filebuffer.conf
deleted file mode 100644
index 77eb3e7..0000000
--- a/streams-contrib/streams-persist-filebuffer/src/main/resources/filebuffer.conf
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-file {
- path = "/tmp/file-queue.txt"
-}
diff --git a/streams-contrib/streams-persist-filebuffer/src/main/resources/reference.conf b/streams-contrib/streams-persist-filebuffer/src/main/resources/reference.conf
deleted file mode 100644
index b2bbd72..0000000
--- a/streams-contrib/streams-persist-filebuffer/src/main/resources/reference.conf
+++ /dev/null
@@ -1,5 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-filebuffer {
- path = "/tmp/streams-file-queue.txt"
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-filebuffer/src/site/markdown/index.md b/streams-contrib/streams-persist-filebuffer/src/site/markdown/index.md
deleted file mode 100644
index 120df24..0000000
--- a/streams-contrib/streams-persist-filebuffer/src/site/markdown/index.md
+++ /dev/null
@@ -1,23 +0,0 @@
-streams-persist-filebuffer
-==========================
-
-Read to / write from File-backed Queue
-
-## Configuration
-
-| Schema |
-|--------|
-| [FileBufferConfiguration.json](org/apache/streams/file/FileBufferConfiguration.json "FileBufferConfiguration.json") [FileBufferConfiguration.html](apidocs/org/apache/streams/file/FileBufferConfiguration.html "javadoc") |
-
-## Components
-
-
-
-| Class | Configuration | Example Configuration(s) |
-|-------|---------------|--------------------------|
-| FileBufferPersistReader [FileBufferPersistReader.html](apidocs/org/apache/streams/file/FileBufferPersistReader.html "javadoc") | [FileBufferReaderConfiguration.json](org/apache/streams/file/FileBufferReaderConfiguration.json "FileBufferReaderConfiguration.json") [FileBufferReaderConfiguration.html](apidocs/org/apache/streams/file/FileBufferReaderConfiguration.html "javadoc") | [filebuffer.conf](filebuffer.conf "filebuffer.conf") |
-| FileBufferPersistWriter [FileBufferPersistWriter.html](apidocs/org/apache/streams/file/FileBufferPersistWriter "javadoc") | [FileBufferWriterConfiguration.json](org/apache/streams/file/FileBufferWriterConfiguration.json "FileBufferWriterConfiguration.json") [FileBufferWriterConfiguration.html](apidocs/org/apache/streams/file/FileBufferWriterConfiguration.html "javadoc") | [filebuffer.conf](filebuffer.conf "filebuffer.conf") |
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-filebuffer/src/test/java/org/apache/streams/filebuffer/test/FileBufferPersistIT.java b/streams-contrib/streams-persist-filebuffer/src/test/java/org/apache/streams/filebuffer/test/FileBufferPersistIT.java
deleted file mode 100644
index 5efd91e..0000000
--- a/streams-contrib/streams-persist-filebuffer/src/test/java/org/apache/streams/filebuffer/test/FileBufferPersistIT.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.filebuffer.test;
-
-import org.apache.streams.console.ConsolePersistReader;
-import org.apache.streams.console.ConsolePersistWriter;
-import org.apache.streams.core.StreamBuilder;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.filebuffer.FileBufferConfiguration;
-import org.apache.streams.filebuffer.FileBufferPersistReader;
-import org.apache.streams.filebuffer.FileBufferPersistWriter;
-import org.apache.streams.local.builders.LocalStreamBuilder;
-
-import org.mockito.Mockito;
-import org.powermock.api.mockito.PowerMockito;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-/**
- * Tests {@link org.apache.streams.filebuffer.FileBufferPersistWriter }
- * Tests {@link org.apache.streams.filebuffer.FileBufferPersistReader }
- */
-public class FileBufferPersistIT {
-
- private FileBufferConfiguration testConfiguration;
-
- private ConsolePersistReader reader = Mockito.mock(ConsolePersistReader.class);
- private ConsolePersistWriter writer = Mockito.mock(ConsolePersistWriter.class);
-
- private StreamsDatum testDatum1 = new StreamsDatum("{\"datum\":1}");
- private StreamsDatum testDatum2 = new StreamsDatum("{\"datum\":2}");
- private StreamsDatum testDatum3 = new StreamsDatum("{\"datum\":3}");
-
- @BeforeClass
- public void prepareTestPersistStream() {
-
- testConfiguration = new FileBufferConfiguration();
- testConfiguration.setPath("target/FilePersistIT.txt");
-
- File file = new File( testConfiguration.getPath());
- if( file.exists() )
- file.delete();
-
- PowerMockito.when(reader.readCurrent())
- .thenReturn(
- new StreamsResultSet(new ConcurrentLinkedQueue<>(
- Arrays.asList(testDatum1, testDatum2, testDatum3)))
- ).thenReturn(null);
- }
-
- @Test
- public void testPersistStream() {
-
- assert(testConfiguration != null);
-
- Map<String, Object> streamConfig = new HashMap<>();
- streamConfig.put(LocalStreamBuilder.TIMEOUT_KEY, 1000);
-
- StreamBuilder builder = new LocalStreamBuilder(1, streamConfig);
-
- FileBufferPersistWriter fileWriter = new FileBufferPersistWriter(testConfiguration);
- FileBufferPersistReader fileReader = new FileBufferPersistReader(testConfiguration);
-
- builder.newReadCurrentStream("stdin", reader);
- builder.addStreamsPersistWriter("writer", fileWriter, 1, "stdin");
- builder.start();
-
- try {
- Thread.sleep(1000);
- } catch (InterruptedException ie) {
- //Handle exception
- }
-
- builder.stop();
-
- builder = new LocalStreamBuilder(1, streamConfig);
- builder.newReadCurrentStream("reader", fileReader);
- builder.addStreamsPersistWriter("stdout", writer, 1, "reader");
-
- builder.start();
-
- try {
- Thread.sleep(1000);
- } catch (InterruptedException ie) {
- //Handle exception
- }
-
- builder.stop();
-
- Mockito.verify(writer).write(testDatum1);
- Mockito.verify(writer).write(testDatum2);
- Mockito.verify(writer).write(testDatum3);
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-filebuffer/src/test/java/org/apache/streams/filebuffer/test/TestFileBufferPersist.java b/streams-contrib/streams-persist-filebuffer/src/test/java/org/apache/streams/filebuffer/test/TestFileBufferPersist.java
deleted file mode 100644
index 677aaf4..0000000
--- a/streams-contrib/streams-persist-filebuffer/src/test/java/org/apache/streams/filebuffer/test/TestFileBufferPersist.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.filebuffer.test;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.filebuffer.FileBufferConfiguration;
-import org.apache.streams.filebuffer.FileBufferPersistReader;
-import org.apache.streams.filebuffer.FileBufferPersistWriter;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Tests @link:{ org.apache.streams.filebuffer.FileBufferWriter }
- * Tests @link:{ org.apache.streams.filebuffer.FileBufferReader }
- */
-public class TestFileBufferPersist {
-
- private FileBufferConfiguration testConfiguration;
-
- @Test
- public void testPersistWriterString() {
-
- testConfiguration = new FileBufferConfiguration();
- testConfiguration.setPath("target/TestFilePersist.txt");
-
- File file = new File( testConfiguration.getPath());
- if( file.exists() )
- file.delete();
-
- FileBufferPersistWriter testPersistWriter = new FileBufferPersistWriter(testConfiguration);
- testPersistWriter.prepare(testConfiguration);
-
- String testJsonString = "{\"dummy\":\"true\"}";
-
- testPersistWriter.write(new StreamsDatum(testJsonString, "test"));
-
- testPersistWriter.cleanUp();
-
- FileBufferPersistReader testPersistReader = new FileBufferPersistReader(testConfiguration);
- try {
- testPersistReader.prepare(testConfiguration);
- } catch( Throwable e ) {
- e.printStackTrace();
- Assert.fail();
- }
-
- StreamsResultSet testResult = testPersistReader.readCurrent();
-
- testPersistReader.cleanUp();
-
- assertEquals(1, testResult.size());
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-graph/README.md b/streams-contrib/streams-persist-graph/README.md
deleted file mode 100644
index 406f6a3..0000000
--- a/streams-contrib/streams-persist-graph/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-persist-graph
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-persist-graph/pom.xml b/streams-contrib/streams-persist-graph/pom.xml
deleted file mode 100644
index ae28c85..0000000
--- a/streams-contrib/streams-persist-graph/pom.xml
+++ /dev/null
@@ -1,169 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-graph</artifactId>
- <name>streams-persist-graph</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- <version>1.2.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-converters</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-http</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.antlr</groupId>
- <artifactId>stringtemplate</artifactId>
- <version>4.0.2</version>
- </dependency>
- <dependency>
- <groupId>org.javatuples</groupId>
- <artifactId>javatuples</artifactId>
- <version>1.2</version>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.graph.pojo</targetPackage>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-http</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- revisit using this if streams bumps to jdk8 -->
- <!--<plugin>-->
- <!--<groupId>com.github.harti2006</groupId>-->
- <!--<artifactId>neo4j-server-maven-plugin</artifactId>-->
- <!--<version>0.1</version>-->
- <!--<configuration>-->
- <!--<port>7474</port>-->
- <!--<version>2.2.1</version>-->
- <!--</configuration>-->
- <!--<executions>-->
- <!--<execution>-->
- <!--<id>start-neo4j-server</id>-->
- <!--<phase>-->
- <!--pre-integration-test-->
- <!--</phase>-->
- <!--<goals>-->
- <!--<goal>start</goal>-->
- <!--</goals>-->
- <!--</execution>-->
- <!--<execution>-->
- <!--<id>stop-neo4j-server</id>-->
- <!--<phase>-->
- <!--post-integration-test-->
- <!--</phase>-->
- <!--<goals>-->
- <!--<goal>stop</goal>-->
- <!--</goals>-->
- <!--</execution>-->
- <!--</executions>-->
- <!--</plugin>-->
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-graph/src/main/java/org/apache/streams/graph/GraphHelper.java b/streams-contrib/streams-persist-graph/src/main/java/org/apache/streams/graph/GraphHelper.java
deleted file mode 100644
index 614dc95..0000000
--- a/streams-contrib/streams-persist-graph/src/main/java/org/apache/streams/graph/GraphHelper.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.graph;
-
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-/**
- * Interface for methods allowing persistance to a graph database wrapped with
- * a rest API. CypherGraphHelper is a good example, for neo4j.
- */
-public interface GraphHelper {
-
- ObjectNode getVertexRequest(String streamsId);
-
- ObjectNode getVertexRequest(Long vertexId);
-
- ObjectNode createVertexRequest(ActivityObject activityObject);
-
- ObjectNode mergeVertexRequest(ActivityObject activityObject);
-
- ObjectNode createEdgeRequest(Activity activity, ActivityObject source, ActivityObject destination);
-
-}
diff --git a/streams-contrib/streams-persist-graph/src/main/java/org/apache/streams/graph/HttpGraphHelper.java b/streams-contrib/streams-persist-graph/src/main/java/org/apache/streams/graph/HttpGraphHelper.java
deleted file mode 100644
index 804e9ff..0000000
--- a/streams-contrib/streams-persist-graph/src/main/java/org/apache/streams/graph/HttpGraphHelper.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.graph;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.javatuples.Pair;
-
-import java.util.Map;
-
-/**
- * Interface for methods allowing persistance to a graph database wrapped with
- * a rest API. CypherGraphHelper is a good example, for neo4j.
- */
-public interface HttpGraphHelper {
-
- ObjectNode readData(Pair<String, Map<String, Object>> queryPlusParameters);
-
- ObjectNode writeData(Pair<String, Map<String, Object>> queryPlusParameters);
-
-}
diff --git a/streams-contrib/streams-persist-graph/src/main/java/org/apache/streams/graph/QueryGraphHelper.java b/streams-contrib/streams-persist-graph/src/main/java/org/apache/streams/graph/QueryGraphHelper.java
deleted file mode 100644
index 38ceb55..0000000
--- a/streams-contrib/streams-persist-graph/src/main/java/org/apache/streams/graph/QueryGraphHelper.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.graph;
-
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import org.javatuples.Pair;
-
-import java.util.Map;
-
-/**
- * Interface for methods allowing persistance to a graph database which uses a combination
- * DSL.
- */
-public interface QueryGraphHelper {
-
- public Pair<String, Map<String, Object>> getVertexRequest(String streamsId);
-
- public Pair<String, Map<String, Object>> getVertexRequest(Long vertexId);
-
- public Pair<String, Map<String, Object>> createVertexRequest(ActivityObject activityObject);
-
- public Pair<String, Map<String, Object>> mergeVertexRequest(ActivityObject activityObject);
-
- public Pair<String, Map<String, Object>> createActorObjectEdge(Activity activity);
-
- public Pair<String, Map<String, Object>> createActorTargetEdge(Activity activity);
-
-}
diff --git a/streams-contrib/streams-persist-graph/src/main/jsonschema/org/apache/streams/graph/GraphReaderConfiguration.json b/streams-contrib/streams-persist-graph/src/main/jsonschema/org/apache/streams/graph/GraphReaderConfiguration.json
deleted file mode 100644
index 6cffba0..0000000
--- a/streams-contrib/streams-persist-graph/src/main/jsonschema/org/apache/streams/graph/GraphReaderConfiguration.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.graph.GraphReaderConfiguration",
- "extends" : {"$ref":"GraphHttpConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "query": {
- "type": "string",
- "required": "true"
- },
- "params": {
- "type": "object"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-graph/src/main/resources/components.dot b/streams-contrib/streams-persist-graph/src/main/resources/components.dot
deleted file mode 100644
index f326cca..0000000
--- a/streams-contrib/streams-persist-graph/src/main/resources/components.dot
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- processors [label="processors", shape="circle"];
-
- subgraph cluster_persisters {
- label="persisters";
- persisters_graph_reader [label="GraphVertexReader"]
- persisters_graph_writer [label="GraphHttpWriter"]
- }
-
- subgraph cluster_dbs {
- label="dbs";
- neo4j [label="neo4j", shape="cylinder"]
- }
-
- generators -> providers
- providers -> processors
- processors -> persisters_graph_writer [label="StreamsDatum"]
- persisters_graph_reader -> processors [label="StreamsDatum"]
- neo4j -> persisters_graph_reader
- persisters_graph_writer -> neo4j
-
-}
diff --git a/streams-contrib/streams-persist-graph/src/main/resources/graph-read.conf b/streams-contrib/streams-persist-graph/src/main/resources/graph-read.conf
deleted file mode 100644
index 802b664..0000000
--- a/streams-contrib/streams-persist-graph/src/main/resources/graph-read.conf
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-{
- "graph": {
- "type": "neo4j",
- "protocol": "http",
- "hostname": "localhost",
- "port": 7474,
- "graph": "data"
- "query": {
- "objects": [
- "actor",
- "object"
- ],
- "verbs": [
- "follow"
- ],
- "objectTypes": [
- "page"
- ]
- }
- }
-}
diff --git a/streams-contrib/streams-persist-graph/src/main/resources/graph-write.conf b/streams-contrib/streams-persist-graph/src/main/resources/graph-write.conf
deleted file mode 100644
index 57f65d9..0000000
--- a/streams-contrib/streams-persist-graph/src/main/resources/graph-write.conf
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-{
- "graph": {
- "type": "neo4j",
- "protocol": "http",
- "hostname": "localhost",
- "port": 7474,
- "graph": "data"
- "vertices": {
- "objects": [
- "actor",
- "object"
- ],
- "verbs": [
- "follow"
- ],
- "objectTypes": [
- "page"
- ]
- }
- }
-}
diff --git a/streams-contrib/streams-persist-graph/src/site/markdown/index.md b/streams-contrib/streams-persist-graph/src/site/markdown/index.md
deleted file mode 100644
index 2987350..0000000
--- a/streams-contrib/streams-persist-graph/src/site/markdown/index.md
+++ /dev/null
@@ -1,23 +0,0 @@
-streams-persist-graph
-=====================
-
-Build a graph index of a stream
-
-## Configuration
-
-| Schema |
-|--------|
-| [GraphConfiguration.json](org/apache/streams/graph/GraphConfiguration.json "GraphConfiguration.json") [GraphConfiguration.html](apidocs/org/apache/streams/graph/GraphConfiguration.html "javadoc") |
-
-## Components
-
-
-
-| Class | Configuration | Example Configuration(s) |
-|-------|---------------|--------------------------|
-| GraphVertexReader [GraphVertexReader.html](apidocs/org/apache/streams/graph/GraphVertexReader.html "javadoc") | [GraphReaderConfiguration.json](org/apache/streams/graph/GraphReaderConfiguration.json "GraphReaderConfiguration.json") [GraphReaderConfiguration.html](apidocs/org/apache/streams/graph/GraphReaderConfiguration.html "javadoc") | [graph-read.conf](graph-read.conf "graph-read.conf") |
-| GraphHttpPersistWriter [GraphHttpPersistWriter.html](apidocs/org/apache/streams/graph/GraphHttpPersistWriter "javadoc") | [GraphHttpConfiguration.json](org/apache/streams/graph/GraphHttpConfiguration.json "GraphHttpConfiguration.json") [GraphHttpConfiguration.html](apidocs/org/apache/streams/graph/GraphHttpConfiguration.html "javadoc") | [graph-write.conf](graph-write.conf "graph-write.conf") |
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-graph/src/site/markdown/neo4j.md b/streams-contrib/streams-persist-graph/src/site/markdown/neo4j.md
deleted file mode 100644
index 7bbe47a..0000000
--- a/streams-contrib/streams-persist-graph/src/site/markdown/neo4j.md
+++ /dev/null
@@ -1,29 +0,0 @@
-## Neo4j
-
-Start neo4j via docker with the docker maven plugin:
-
- mvn -PdockerITs docker:start
-
-Confirm that neo4j is running:
-
- docker ps
-
-Confirm that host and post(s) are in property file:
-
- cat neo4j.properties
-
-Create a local file `neo4j.conf` with cluster details:
-
- neo4j {
- hostname = ${neo4j.tcp.host}
- port = ${neo4j.tcp.port}
- type = "neo4j"
- graph = "data"
- }
-
-When configuring a stream, include these files:
-
- include "neo4j.properties"
- include "neo4j.conf"
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-graph/src/site/site.xml b/streams-contrib/streams-persist-graph/src/site/site.xml
deleted file mode 100644
index b894666..0000000
--- a/streams-contrib/streams-persist-graph/src/site/site.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project>
- <body>
- <links name="Help">
- <item name="Neo4j" href="neo4j.html"/>
- </links>
- </body>
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-graph/src/test/resources/sampleReaderResult.json b/streams-contrib/streams-persist-graph/src/test/resources/sampleReaderResult.json
deleted file mode 100644
index 7b65f0c..0000000
--- a/streams-contrib/streams-persist-graph/src/test/resources/sampleReaderResult.json
+++ /dev/null
@@ -1,119 +0,0 @@
-{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "columns": [
- "v"
- ],
- "data": [
- [
- {
- "data": {
- "extensions.screenName": "*@^$&!",
- "extensions.location": "Silicon Valley",
- "extensions.posts": 10134,
- "extensions.favorites": 5468,
- "extensions.followers": 20062
- }
- }
- ],
- [
- {
- "data": {
- "extensions.screenName": ")#(%&!",
- "extensions.location": "Chicago",
- "extensions.posts": 44728,
- "extensions.favorites": 7759,
- "extensions.followers": 28551
- }
- }
- ],
- [
- {
- "data": {
- "extensions.screenName": "(@&)!*$&@",
- "extensions.location": "Philadelphia",
- "extensions.posts": 12725,
- "extensions.favorites": 782,
- "extensions.followers": 457
- }
- }
- ],
- [
- {
- "data": {
- "extensions.screenName": "*(%&)@(#**$@!",
- "extensions.location": "San Francisco",
- "extensions.posts": 14847,
- "extensions.favorites": 7168,
- "extensions.followers": 20349
- }
- }
- ],
- [
- {
- "data": {
- "extensions.screenName": "&%#*(@#$)(*&^@#$&^%#",
- "extensions.location": "London|Spain|USA|Asia|GLOBAL",
- "extensions.posts": 82955,
- "extensions.favorites": 2882,
- "extensions.followers": 120109
- }
- }
- ],
- [
- {
- "data": {
- "extensions.screenName": "^$**!",
- "extensions.location": "",
- "extensions.posts": 34941,
- "extensions.favorites": 87987,
- "extensions.followers": 1503801
- }
- }
- ],
- [
- {
- "data": {
- "extensions.screenName": "*#(&%#*(&@#$*&!",
- "extensions.location": "Austin, TX",
- "extensions.posts": 27400,
- "extensions.favorites": 0,
- "extensions.followers": 16695
- }
- }
- ],
- [
- {
- "data": {
- "extensions.screenName": "$#*&(%(*#)@#$",
- "extensions.location": "Silicon Valley, Ca.",
- "extensions.posts": 50558,
- "extensions.favorites": 2408,
- "extensions.followers": 115395
- }
- }
- ],
- [
- {
- "data": {
- "extensions.screenName": "#5&*($@(#*&",
- "extensions.location": "Online/IRL",
- "extensions.posts": 97369,
- "extensions.favorites": 13677,
- "extensions.followers": 210024
- }
- }
- ],
- [
- {
- "data": {
- "extensions.screenName": "3%&*#$&*((P)@",
- "extensions.location": "Austinite Abroad in London",
- "extensions.posts": 32414,
- "extensions.favorites": 1305,
- "extensions.followers": 6997
- }}
- ]
- ]
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-hbase/README.md b/streams-contrib/streams-persist-hbase/README.md
deleted file mode 100644
index d85c605..0000000
--- a/streams-contrib/streams-persist-hbase/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-persist-hbase
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-persist-hbase/pom.xml b/streams-contrib/streams-persist-hbase/pom.xml
deleted file mode 100644
index f31eedf..0000000
--- a/streams-contrib/streams-persist-hbase/pom.xml
+++ /dev/null
@@ -1,210 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-hbase</artifactId>
- <name>${project.artifactId}</name>
-
- <description>HBase Module</description>
-
- <properties>
- <hadoop-common.version>2.7.0</hadoop-common.version>
- <hbase.version>1.2.3</hbase.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop-common.version}</version>
- <scope>compile</scope>
- <type>jar</type>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${hbase.version}</version>
- <type>jar</type>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <version>2.5.0</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.hbase.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <id>dockerITs</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>skipITs</name>
- <value>false</value>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>io.fabric8</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- <configuration combine.self="override">
- <watchInterval>500</watchInterval>
- <logDate>default</logDate>
- <verbose>true</verbose>
- <autoPull>on</autoPull>
- <images>
- <image>
- <name>nerdammer/hbase</name>
- <alias>hbase</alias>
- <run>
- <namingStrategy>none</namingStrategy>
- <ports>
- <port>${zookeeper.tcp.host}:${zookeeper.tcp.port}:2181</port>
- <port>${hbase.tcp.host}:${hbase.tcp.port}:60000</port>
- <port>${hbase.http.host}:${hbase.http.port}:60010</port>
- </ports>
- <portPropertyFile>hbase.properties</portPropertyFile> -->
- <wait>
- <log>hbase startup</log>
- <http>
- <url>http://${hbase.http.host}:${hbase.http.port}</url>
- <method>GET</method>
- <status>200</status>
- </http>
- <time>20000</time>
- <kill>1000</kill>
- <shutdown>500</shutdown>
- </wait>
- <log>
- <enabled>true</enabled>
- <date>default</date>
- <color>cyan</color>
- </log>
- </run>
- <watch>
- <mode>none</mode>
- </watch>
- </image>
-
- </images>
- </configuration>
-
- </plugin>
-
- </plugins>
- </build>
-
- </profile>
- </profiles>
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistReader.java b/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistReader.java
deleted file mode 100644
index 3527cfb..0000000
--- a/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistReader.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hbase;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.Properties;
-import java.util.Queue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-public class HbasePersistReader implements StreamsPersistReader, Serializable {
-
- public final static String STREAMS_ID = "HbasePersistReader";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(HbasePersistReader.class);
-
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- protected final AtomicBoolean running = new AtomicBoolean();
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = new ObjectMapper();
-
- protected HbaseConfiguration config;
-
- protected Properties props = new Properties();
-
- private ExecutorService executor = Executors.newSingleThreadExecutor();
-
- public HbasePersistReader() {
- this.config = new ComponentConfigurator<>(HbaseConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("hbase"));
- }
-
- private HbasePersistReaderTask task;
-
- public HbasePersistReader(HbaseConfiguration config) {
- this.config = config;
- }
-
- public void setConfig(HbaseConfiguration config) {
- this.config = config;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
-
- LOGGER.debug("{} startStream", STREAMS_ID);
-
- executor.submit(task);
-
- running.set(true);
-
- }
-
- @Override
- public StreamsResultSet readAll() {
- return readCurrent();
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet result;
-
- LOGGER.debug("Providing {} docs", persistQueue.size());
-
- try {
- lock.readLock().lock();
- result = new StreamsResultSet(persistQueue);
- persistQueue = constructQueue();
- } finally {
- lock.readLock().unlock();
- }
-
- return result;
- }
-
- protected Queue<StreamsDatum> constructQueue() {
- return new LinkedBlockingQueue<StreamsDatum>();
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger bigInteger) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime dateTime, DateTime dateTime2) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return running.get();
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
-// props.put("bootstrap.servers", config.getBrokerlist());
-// props.put("group.id", config.getGroupId());
-// props.put("auto.commit.interval.ms", "1000");
-// props.put("auto.offset.reset", "earliest");
-// props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
-// props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
-
- persistQueue = constructQueue();
-
- task = new HbasePersistReaderTask(this);
-
- }
-
- @Override
- public void cleanUp() {
- running.set(false);
- executor.shutdown();
-//
-// try {
-// executor.awaitTermination(5, TimeUnit.SECONDS);
-// } catch (InterruptedException ignored) {}
-
- }
-
- public void write(StreamsDatum streamsDatum) {
- try {
- lock.writeLock().lock();
- persistQueue.add(streamsDatum);
- } finally {
- lock.writeLock().unlock();
- }
-
- }
-}
diff --git a/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistReaderTask.java b/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistReaderTask.java
deleted file mode 100644
index 0fafd47..0000000
--- a/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistReaderTask.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hbase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HbasePersistReaderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(HbasePersistReaderTask.class);
-
- private HbasePersistReader reader;
-
- public HbasePersistReaderTask(HbasePersistReader reader) {
- this.reader = reader;
- }
-
- @Override
- public void run() {
-
- while(reader.isRunning()) {
-
-
- }
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriter.java b/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriter.java
deleted file mode 100644
index 65bf662..0000000
--- a/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriter.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hbase;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.util.GuidUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.HTablePool;
-import org.apache.hadoop.hbase.client.Put;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.Flushable;
-import java.io.IOException;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-/**
- * HbasePersistWriter writes to hbase.
- */
-public class HbasePersistWriter implements StreamsPersistWriter, Flushable, Closeable {
-
- public static final String STREAMS_ID = "HbasePersistWriter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(HbasePersistWriter.class);
-
- protected HConnection connection;
- protected HTablePool pool;
- protected HTableInterface table;
- protected HTableDescriptor descriptor;
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = new ObjectMapper();
-
- private HbaseConfiguration config;
-
- /**
- * HbasePersistWriter constructor - resolve HbaseConfiguration from JVM 'hbase'.
- */
- public HbasePersistWriter() {
- this.config = new ComponentConfigurator<>(HbaseConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("hbase"));
- this.persistQueue = new ConcurrentLinkedQueue<>();
- }
-
- /**
- * HbasePersistWriter constructor - use supplied persistQueue.
- * @param hbaseConfiguration HbaseConfiguration
- */
- // TODO: refactor this to use HbaseConfiguration
- public HbasePersistWriter(HbaseConfiguration hbaseConfiguration) {
- this.config = hbaseConfiguration;
- this.persistQueue = new ConcurrentLinkedQueue<>();
- }
-
- private synchronized void connectToHbase() {
-
- // TODO: refactor this to resolve this stuff from typesafe
- Configuration configuration = new Configuration();
- configuration.set("hbase.rootdir", config.getRootdir());
- configuration.set("zookeeper.znode.parent", config.getParent());
- configuration.set("zookeeper.znode.rootserver", config.getRootserver());
- configuration.set("hbase.cluster.distributed", "false");
- configuration.set("hbase.zookeeper.quorum", config.getQuorum());
- configuration.set("hbase.zookeeper.property.clientPort", Long.toString(config.getClientPort()));
- configuration.setInt("zookeeper.session.timeout", 1000);
-
- configuration.setInt("timeout", 1000);
-
- //pool = new HTablePool(configuration, 10);
- try {
- connection = HConnectionManager.createConnection(configuration);
- } catch (Exception ex) {
- ex.printStackTrace();
- return;
- }
-
- try {
- // table = new HTable(configuration, config.getTable());
- // table = (HTable) pool.getTable(config.getTable());
- table = new HTable(configuration, config.getTable().getBytes());
- table.setAutoFlush(true);
- } catch (Exception ex) {
- ex.printStackTrace();
- return;
- }
- //
-
- try {
- descriptor = table.getTableDescriptor();
- } catch (Exception ex) {
- ex.printStackTrace();
- return;
- }
-
- try {
- LOGGER.info("Table : {}", descriptor);
- } catch (Exception ex) {
- LOGGER.error("There was an error connecting to HBase, please check your settings and try again");
- ex.printStackTrace();
- }
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void write(StreamsDatum streamsDatum) {
-
- ObjectNode node;
- byte[] row;
- if (StringUtils.isNotBlank(streamsDatum.getId())) {
- row = streamsDatum.getId().getBytes();
- } else {
- row = GuidUtils.generateGuid(streamsDatum.toString()).getBytes();
- }
- Put put = new Put(row);
- if ( streamsDatum.getDocument() instanceof String ) {
- try {
- node = mapper.readValue((String)streamsDatum.getDocument(), ObjectNode.class);
- } catch (IOException ex) {
- ex.printStackTrace();
- LOGGER.warn("Invalid json: {}", streamsDatum.getDocument().toString());
- return;
- }
- try {
- byte[] value = node.binaryValue();
- put.add(config.getFamily().getBytes(), config.getQualifier().getBytes(), value);
- } catch (IOException ex) {
- ex.printStackTrace();
- LOGGER.warn("Failure adding object: {}", streamsDatum.getDocument().toString());
- return;
- }
- } else {
- try {
- node = mapper.valueToTree(streamsDatum.getDocument());
- } catch (Exception ex) {
- ex.printStackTrace();
- LOGGER.warn("Invalid json: {}", streamsDatum.getDocument().toString());
- return;
- }
- put.setId(GuidUtils.generateGuid(node.toString()));
- try {
- byte[] value = node.binaryValue();
- put.add(config.getFamily().getBytes(), config.getQualifier().getBytes(), value);
- } catch (IOException ex) {
- ex.printStackTrace();
- LOGGER.warn("Failure preparing put: {}", streamsDatum.getDocument().toString());
- return;
- }
-
- }
- try {
- table.put(put);
- } catch (IOException ex) {
- ex.printStackTrace();
- LOGGER.warn("Failure executin put: {}", streamsDatum.getDocument().toString());
- }
-
- }
-
- public void flush() throws IOException {
- table.flushCommits();
- }
-
- public synchronized void close() throws IOException {
- table.close();
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- connectToHbase();
-
- Thread task = new Thread(new HbasePersistWriterTask(this));
- task.start();
-
- try {
- task.join();
- } catch (InterruptedException ex) {
- ex.printStackTrace();
- }
-
- }
-
- @Override
- public void cleanUp() {
-
- try {
- flush();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- try {
- close();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
-
- }
-}
diff --git a/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriterTask.java b/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriterTask.java
deleted file mode 100644
index eef7004..0000000
--- a/streams-contrib/streams-persist-hbase/src/main/java/org/apache/streams/hbase/HbasePersistWriterTask.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hbase;
-
-import org.apache.streams.core.StreamsDatum;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Random;
-
-/**
- * HbasePersistWriterTask writes to hbase on behalf of
- * @see org.apache.streams.hbase.HbasePersistWriter
- */
-public class HbasePersistWriterTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(HbasePersistWriterTask.class);
-
- private HbasePersistWriter writer;
-
- public HbasePersistWriterTask(HbasePersistWriter writer) {
- this.writer = writer;
- }
-
- @Override
- public void run() {
-
- while (true) {
- if ( writer.persistQueue.peek() != null ) {
- try {
- StreamsDatum entry = writer.persistQueue.remove();
- writer.write(entry);
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
- try {
- Thread.sleep(new Random().nextInt(1));
- } catch (InterruptedException ignored) {
- LOGGER.trace("ignored InterruptedException", ignored);
- }
- }
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-hbase/src/main/jsonschema/org/apache/streams/hbase/HbaseConfiguration.json b/streams-contrib/streams-persist-hbase/src/main/jsonschema/org/apache/streams/hbase/HbaseConfiguration.json
deleted file mode 100644
index f531fa9..0000000
--- a/streams-contrib/streams-persist-hbase/src/main/jsonschema/org/apache/streams/hbase/HbaseConfiguration.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.hbase.HbaseConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "rootdir": {
- "type": "string",
- "description": "Hbase host"
- },
- "parent": {
- "type": "string",
- "description": "WebHdfs port"
- },
- "rootserver": {
- "type": "string",
- "description": "Base path"
- },
- "quorum": {
- "type": "string",
- "description": "User"
- },
- "clientPort": {
- "type": "integer",
- "description": "ZK Port"
- },
- "table": {
- "type": "string",
- "description": "Table"
- },
- "family": {
- "type": "string",
- "description": "Column Family"
- },
- "qualifier": {
- "type": "string",
- "description": "Qualifier"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-hbase/src/main/resources/components.dot b/streams-contrib/streams-persist-hbase/src/main/resources/components.dot
deleted file mode 100644
index 62e4996..0000000
--- a/streams-contrib/streams-persist-hbase/src/main/resources/components.dot
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- processors [label="processors", shape="circle"];
-
- subgraph cluster_persisters {
- label="persisters";
- persisters_hbase_writer [label="HbasePersistWriter"]
- }
-
- subgraph cluster_dbs {
- label="dbs";
- hbase [label="hbase", shape="cylinder"]
- }
-
- generators -> providers
- providers -> processors
- processors -> persisters_hbase_writer [label="StreamsDatum[String]\nStreamsDatum[ObjectNode]"]
- persisters_hbase_writer -> hbase
-
-}
diff --git a/streams-contrib/streams-persist-hbase/src/main/resources/hbase.conf b/streams-contrib/streams-persist-hbase/src/main/resources/hbase.conf
deleted file mode 100644
index 7993177..0000000
--- a/streams-contrib/streams-persist-hbase/src/main/resources/hbase.conf
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-hbase {
- rootdir = "hdfs://cloudera-vm-local:8020/hbase"
- zookeeper {
- quorum = "cloudera-vm-local"
- property {
- clientPort = 2181
- }
- }
- table = "test_table"
- family = "test_family"
- qualifier = "test_column"
-}
diff --git a/streams-contrib/streams-persist-hbase/src/main/resources/reference.properties b/streams-contrib/streams-persist-hbase/src/main/resources/reference.properties
deleted file mode 100644
index 699f655..0000000
--- a/streams-contrib/streams-persist-hbase/src/main/resources/reference.properties
+++ /dev/null
@@ -1,10 +0,0 @@
-hbase.rootdir = "hdfs://localhost:8020/hbase"
-
-zookeeper.znode.parent = "/hbase"
-
-zookeeper.znode.rootserver = "localhost"
-
-hbase.zookeeper.quorum = "localhost"
-
-hbase.zookeeper.property.clientPort = 2181
-
diff --git a/streams-contrib/streams-persist-hbase/src/site/markdown/index.md b/streams-contrib/streams-persist-hbase/src/site/markdown/index.md
deleted file mode 100644
index fa30173..0000000
--- a/streams-contrib/streams-persist-hbase/src/site/markdown/index.md
+++ /dev/null
@@ -1,22 +0,0 @@
-streams-persist-hbase
-=====================
-
-Read and write to HBase
-
-## Configuration
-
-| Schema |
-|--------|
-| [HbaseConfiguration.json](org/apache/streams/hbase/HbaseConfiguration.json "HbaseConfiguration.json") [HbaseConfiguration.html](apidocs/org/apache/streams/hbase/HbaseConfiguration.html "javadoc") |
-
-## Components
-
-
-
-| Class | Configuration | Example Configuration(s) |
-|-------|---------------|--------------------------|
-| HbasePersistWriter [HbasePersistWriter.html](apidocs/org/apache/streams/hbase/HbasePersistWriter "javadoc") | [HbaseConfiguration.json](org/apache/streams/hbase/HbaseConfiguration.json "HbaseConfiguration.json") [HbaseConfiguration.html](apidocs/org/apache/streams/hbase/HbaseConfiguration.html "javadoc") | [hbase.conf](hbase.conf "hbase.conf") |
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-hbase/src/test/java/org/apache/streams/hbase/test/HbasePersistReaderIT.java b/streams-contrib/streams-persist-hbase/src/test/java/org/apache/streams/hbase/test/HbasePersistReaderIT.java
deleted file mode 100644
index b514b56..0000000
--- a/streams-contrib/streams-persist-hbase/src/test/java/org/apache/streams/hbase/test/HbasePersistReaderIT.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hbase.test;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.hbase.HbaseConfiguration;
-import org.apache.streams.hbase.HbasePersistReader;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Test HbasePersistReader.
- */
-@Test
- (
- groups={"HbasePersistReaderIT"},
- dependsOnGroups={"HbasePersistWriterIT"}
- )
-public class HbasePersistReaderIT {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(HbasePersistReaderIT.class);
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- protected HbaseConfiguration testConfiguration;
-
- @BeforeClass
- public void prepareTest() throws Exception {
-
- Config reference = ConfigFactory.load();
- File conf_file = new File("target/test-classes/HbasePersistReaderIT.conf");
- assert(conf_file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(HbaseConfiguration.class).detectConfiguration(typesafe, "hbase");
-
- }
-
- @Test(enabled = false)
- public void testPersistReader() throws Exception {
-
- HbasePersistReader testPersistReader = new HbasePersistReader(testConfiguration);
- testPersistReader.prepare(testConfiguration);
- testPersistReader.startStream();
-
- Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS);
-
- testPersistReader.cleanUp();
-
- while( testPersistReader.isRunning() ) {
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
- }
-
- StreamsResultSet resultSet = testPersistReader.readCurrent();
-
- assert( resultSet.size() > 0 );
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-hbase/src/test/java/org/apache/streams/hbase/test/HbasePersistWriterIT.java b/streams-contrib/streams-persist-hbase/src/test/java/org/apache/streams/hbase/test/HbasePersistWriterIT.java
deleted file mode 100644
index 5b8c7f2..0000000
--- a/streams-contrib/streams-persist-hbase/src/test/java/org/apache/streams/hbase/test/HbasePersistWriterIT.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hbase.test;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.hbase.HbaseConfiguration;
-import org.apache.streams.hbase.HbasePersistWriter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import static org.testng.Assert.assertTrue;
-
-/**
- * Test HbasePersistWriterIT.
- */
-@Test
- (
- groups={"HbasePersistWriterIT"}
- )
-public class HbasePersistWriterIT {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(HbasePersistWriterIT.class);
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- protected HbaseConfiguration testConfiguration;
-
- @BeforeClass
- public void prepareTest() throws Exception {
-
- Config reference = ConfigFactory.load();
- File conf_file = new File("target/test-classes/HbasePersistWriterIT.conf");
- assertTrue(conf_file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(HbaseConfiguration.class).detectConfiguration(typesafe, "hbase");
-
- }
-
- @Test(enabled = false)
- public void testPersistWriter() throws Exception {
-
- HbasePersistWriter testPersistWriter = new HbasePersistWriter(testConfiguration);
- testPersistWriter.prepare(testConfiguration);
-
- InputStream testActivityFolderStream = HbasePersistWriterIT.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = HbasePersistWriterIT.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- testPersistWriter.write( datum );
- LOGGER.info("Wrote: " + activity.getVerb() );
- }
-
- testPersistWriter.cleanUp();
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-hbase/src/test/resources/HbasePersistReaderIT.conf b/streams-contrib/streams-persist-hbase/src/test/resources/HbasePersistReaderIT.conf
deleted file mode 100644
index 8e073f6..0000000
--- a/streams-contrib/streams-persist-hbase/src/test/resources/HbasePersistReaderIT.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-hbase {
- rootdir = "/"
- parent = "/"
- rootserver = "/"
- quorum = ${hbase.tcp.host}
- clientPort = ${hbase.tcp.port}
- table = "test_table"
- family = "test_family"
- qualifier = "test_column"
-}
diff --git a/streams-contrib/streams-persist-hbase/src/test/resources/HbasePersistWriterIT.conf b/streams-contrib/streams-persist-hbase/src/test/resources/HbasePersistWriterIT.conf
deleted file mode 100644
index 8e073f6..0000000
--- a/streams-contrib/streams-persist-hbase/src/test/resources/HbasePersistWriterIT.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-hbase {
- rootdir = "/"
- parent = "/"
- rootserver = "/"
- quorum = ${hbase.tcp.host}
- clientPort = ${hbase.tcp.port}
- table = "test_table"
- family = "test_family"
- qualifier = "test_column"
-}
diff --git a/streams-contrib/streams-persist-hdfs/README.md b/streams-contrib/streams-persist-hdfs/README.md
deleted file mode 100644
index 35006d1..0000000
--- a/streams-contrib/streams-persist-hdfs/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-persist-hdfs
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-persist-hdfs/pom.xml b/streams-contrib/streams-persist-hdfs/pom.xml
deleted file mode 100644
index 88cf298..0000000
--- a/streams-contrib/streams-persist-hdfs/pom.xml
+++ /dev/null
@@ -1,213 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-hdfs</artifactId>
- <name>${project.artifactId}</name>
-
- <description>HDFS Module</description>
-
- <properties>
- <hdfs.version>2.7.0</hdfs.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-converters</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hdfs.version}</version>
- <scope>compile</scope>
- <type>jar</type>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hdfs.version}</version>
- <scope>compile</scope>
- <type>jar</type>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hdfs.version}</version>
- <type>pom</type>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-httpfs</artifactId>
- <version>${hdfs.version}</version>
- <type>pom</type>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.hdfs</targetPackage>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-converters</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>resource-dependencies</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/HdfsConstants.java b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/HdfsConstants.java
deleted file mode 100644
index b6660ab..0000000
--- a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/HdfsConstants.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hdfs;
-
-/**
- * Predefined field symbols for streams-persist-hdfs.
- */
-public class HdfsConstants {
-
- protected static final String ID = "ID";
- protected static final String TS = "TS";
- protected static final String META = "META";
- protected static final String DOC = "DOC";
-
-}
diff --git a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java
deleted file mode 100644
index a673d8f..0000000
--- a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hdfs;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.converter.LineReadWriteUtil;
-import org.apache.streams.core.DatumStatusCountable;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Queues;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.math.BigInteger;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.security.PrivilegedExceptionAction;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-
-/**
- * WebHdfsPersistReader reads from hdfs.
- */
-public class WebHdfsPersistReader implements StreamsPersistReader, DatumStatusCountable {
-
- public static final String STREAMS_ID = "WebHdfsPersistReader";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(WebHdfsPersistReader.class);
-
- protected static final char DELIMITER = '\t';
-
- protected FileSystem client;
- protected Path path;
- protected FileStatus[] status;
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- protected ObjectMapper mapper;
- protected LineReadWriteUtil lineReaderUtil;
-
- protected HdfsReaderConfiguration hdfsConfiguration;
- protected StreamsConfiguration streamsConfiguration;
-
- private ExecutorService executor;
-
- protected DatumStatusCounter countersTotal = new DatumStatusCounter();
- protected DatumStatusCounter countersCurrent = new DatumStatusCounter();
- private Future<?> task;
-
- /**
- * WebHdfsPersistReader constructor - resolves HdfsReaderConfiguration from JVM 'hdfs'.
- */
- public WebHdfsPersistReader() {
- this(new ComponentConfigurator<>(HdfsReaderConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig().getConfig("hdfs")));
- }
-
- /**
- * WebHdfsPersistReader constructor - uses supplied HdfsReaderConfiguration.
- * @param hdfsConfiguration hdfsConfiguration
- */
- public WebHdfsPersistReader(HdfsReaderConfiguration hdfsConfiguration) {
- this.hdfsConfiguration = hdfsConfiguration;
- }
-
- /**
- * getURI from hdfsConfiguration.
- * @return URI
- * @throws URISyntaxException URISyntaxException
- */
- public URI getURI() throws URISyntaxException {
- StringBuilder uriBuilder = new StringBuilder();
- uriBuilder.append(hdfsConfiguration.getScheme());
- uriBuilder.append("://");
- if (StringUtils.isNotBlank(hdfsConfiguration.getHost())) {
- uriBuilder.append(hdfsConfiguration.getHost());
- if (hdfsConfiguration.getPort() != null) {
- uriBuilder.append(":" + hdfsConfiguration.getPort());
- }
- } else {
- uriBuilder.append("/");
- }
- return new URI(uriBuilder.toString());
- }
-
- /**
- * isConnected.
- * @return true if connected, false otherwise
- */
- public boolean isConnected() {
- return (client != null);
- }
-
- /**
- * getFileSystem.
- * @return FileSystem
- */
- public final synchronized FileSystem getFileSystem() {
- // Check to see if we are connected.
- if (!isConnected()) {
- connectToWebHDFS();
- }
- return this.client;
- }
-
- // TODO: combine with WebHdfsPersistReader.connectToWebHDFS
- private synchronized void connectToWebHDFS() {
- try {
- LOGGER.info("User : {}", this.hdfsConfiguration.getUser());
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser(this.hdfsConfiguration.getUser());
- ugi.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.SIMPLE);
-
- ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
- Configuration conf = new Configuration();
- conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- conf.set("fs.hdfs.impl", DistributedFileSystem.class.getName());
- conf.set("fs.file.impl", LocalFileSystem.class.getName());
- LOGGER.info("WebURI : {}", getURI().toString());
- client = FileSystem.get(getURI(), conf);
- LOGGER.info("Connected to WebHDFS");
-
- /*
- * ************************************************************************************************
- * This code is an example of how you would work with HDFS and you weren't going over
- * the webHDFS protocol.
- *
- * Smashew: 2013-10-01
- * ************************************************************************************************
- conf.set("fs.defaultFS", "hdfs://hadoop.mdigitallife.com:8020/user/" + userName);
- conf.set("namenode.host","0.0.0.0");
- conf.set("hadoop.job.ugi", userName);
- conf.set(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, "runner");
- fileSystem.createNewFile(new Path("/user/"+ userName + "/test"));
- FileStatus[] status = fs.listStatus(new Path("/user/" + userName));
- for(int i=0;i<status.length;i++)
- {
- LOGGER.info("Directory: {}", status[i].getPath());
- }
- */
- return null;
- });
- } catch (Exception ex) {
- LOGGER.error("There was an error connecting to WebHDFS, please check your settings and try again");
- ex.printStackTrace();
- }
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- LOGGER.debug("Prepare");
- lineReaderUtil = LineReadWriteUtil.getInstance(hdfsConfiguration);
- connectToWebHDFS();
- String pathString = hdfsConfiguration.getPath() + "/" + hdfsConfiguration.getReaderPath();
- LOGGER.info("Path : {}", pathString);
- path = new Path(pathString);
- try {
- if ( client.isFile(path)) {
- LOGGER.info("Found File");
- FileStatus fileStatus = client.getFileStatus(path);
- status = new FileStatus[1];
- status[0] = fileStatus;
- } else if ( client.isDirectory(path)) {
- status = client.listStatus(path);
- List<FileStatus> statusList = Arrays.asList(status);
- Collections.sort(statusList);
- status = statusList.toArray(new FileStatus[0]);
- LOGGER.info("Found Directory : {} files", status.length);
- } else {
- LOGGER.error("Neither file nor directory, wtf");
- }
- } catch (IOException ex) {
- LOGGER.error("IOException", ex);
- }
- streamsConfiguration = StreamsConfigurator.detectConfiguration();
- persistQueue = Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(streamsConfiguration.getBatchSize().intValue()));
- executor = Executors.newSingleThreadExecutor();
- mapper = StreamsJacksonMapper.getInstance();
- }
-
- @Override
- public void cleanUp() {
-
- }
-
- @Override
- public StreamsResultSet readAll() {
- WebHdfsPersistReaderTask readerTask = new WebHdfsPersistReaderTask(this);
- Thread readerThread = new Thread(readerTask);
- readerThread.start();
- try {
- readerThread.join();
- } catch (InterruptedException ignored) {
- LOGGER.trace("ignored InterruptedException", ignored);
- }
- return new StreamsResultSet(persistQueue);
- }
-
- @Override
- public void startStream() {
- LOGGER.debug("startStream");
- task = executor.submit(new WebHdfsPersistReaderTask(this));
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- synchronized ( WebHdfsPersistReader.class ) {
- current = new StreamsResultSet(new ConcurrentLinkedQueue<>(persistQueue));
- current.setCounter(new DatumStatusCounter());
- current.getCounter().add(countersCurrent);
- countersTotal.add(countersCurrent);
- countersCurrent = new DatumStatusCounter();
- persistQueue.clear();
- }
-
- return current;
- }
-
- protected void write( StreamsDatum entry ) {
- boolean success;
- do {
- synchronized ( WebHdfsPersistReader.class ) {
- success = persistQueue.offer(entry);
- }
- Thread.yield();
- }
- while ( !success );
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return task == null || !task.isDone() && !task.isCancelled();
- }
-
- @Override
- public DatumStatusCounter getDatumStatusCounter() {
- return countersTotal;
- }
-}
diff --git a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReaderTask.java b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReaderTask.java
deleted file mode 100644
index fa287eb..0000000
--- a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReaderTask.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hdfs;
-
-import org.apache.streams.core.DatumStatus;
-import org.apache.streams.core.StreamsDatum;
-
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.fs.FileStatus;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.concurrent.TimeUnit;
-import java.util.zip.GZIPInputStream;
-
-/**
- * WebHdfsPersistReaderTask reads from hdfs on behalf of
- * @see org.apache.streams.hdfs.WebHdfsPersistReader
- */
-public class WebHdfsPersistReaderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(WebHdfsPersistReaderTask.class);
-
- private WebHdfsPersistReader reader;
-
- public WebHdfsPersistReaderTask(WebHdfsPersistReader reader) {
- this.reader = reader;
- }
-
- @Override
- public void run() {
-
- LOGGER.info("WebHdfsPersistReaderTask: files to process");
-
- for ( FileStatus fileStatus : reader.status ) {
- LOGGER.info(" " + fileStatus.getPath().getName());
- }
-
- for ( FileStatus fileStatus : reader.status ) {
- InputStream inputStream;
- InputStreamReader inputStreamReader;
- BufferedReader bufferedReader;
- if ( fileStatus.isFile() && !fileStatus.getPath().getName().startsWith("_")) {
- HdfsWriterConfiguration.Compression compression = HdfsWriterConfiguration.Compression.NONE;
- if ( fileStatus.getPath().getName().endsWith(".gz")) {
- compression = HdfsWriterConfiguration.Compression.GZIP;
- }
- LOGGER.info("Started Processing: {} Encoding: {} Compression: {}", fileStatus.getPath().getName(), reader.hdfsConfiguration.getEncoding(), compression.toString());
- try {
- inputStream = reader.client.open(fileStatus.getPath());
- if ( compression.equals(HdfsWriterConfiguration.Compression.GZIP)) {
- inputStream = new GZIPInputStream(inputStream);
- }
- inputStreamReader = new InputStreamReader(inputStream, reader.hdfsConfiguration.getEncoding());
- bufferedReader = new BufferedReader(inputStreamReader);
- } catch (Exception ex) {
- LOGGER.error("Exception Opening " + fileStatus.getPath(), ex.getMessage());
- return;
- }
-
- String line = "";
- do {
- try {
- line = bufferedReader.readLine();
- if (StringUtils.isNotEmpty(line)) {
- reader.countersCurrent.incrementAttempt();
- StreamsDatum entry = reader.lineReaderUtil.processLine(line);
- if ( entry != null ) {
- reader.write(entry);
- reader.countersCurrent.incrementStatus(DatumStatus.SUCCESS);
- } else {
- LOGGER.warn("processLine failed");
- reader.countersCurrent.incrementStatus(DatumStatus.FAIL);
- }
- }
- } catch (Exception ex) {
- LOGGER.warn("WebHdfsPersistReader readLine Exception: {}", ex);
- reader.countersCurrent.incrementStatus(DatumStatus.FAIL);
- }
- }
- while (StringUtils.isNotEmpty(line));
- LOGGER.info("Finished Processing " + fileStatus.getPath().getName());
- try {
- bufferedReader.close();
- } catch (Exception ex) {
- LOGGER.error("WebHdfsPersistReader Exception: {}", ex);
- }
- }
- }
-
- LOGGER.info("WebHdfsPersistReaderTask Finished");
-
- Uninterruptibles.sleepUninterruptibly(15, TimeUnit.SECONDS);
- }
-
-}
diff --git a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistWriter.java b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistWriter.java
deleted file mode 100644
index 9079c7e..0000000
--- a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistWriter.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hdfs;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.converter.LineReadWriteUtil;
-import org.apache.streams.core.DatumStatus;
-import org.apache.streams.core.DatumStatusCountable;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.Flushable;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Queue;
-import java.util.zip.GZIPOutputStream;
-
-/**
- * WebHdfsPersistWriter writes to hdfs.
- */
-public class WebHdfsPersistWriter implements StreamsPersistWriter, Flushable, Closeable, DatumStatusCountable {
-
- public static final String STREAMS_ID = "WebHdfsPersistWriter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(WebHdfsPersistWriter.class);
-
- private FileSystem client;
- private Path path;
- private int linesPerFile;
- private int totalRecordsWritten = 0;
- private final List<Path> writtenFiles = new ArrayList<>();
- private int fileLineCounter = 0;
- private OutputStreamWriter currentWriter = null;
-
- private static final int BYTES_IN_MB = 1024 * 1024;
- private static final int BYTES_BEFORE_FLUSH = 64 * BYTES_IN_MB;
- private volatile int totalByteCount = 0;
- private volatile int byteCount = 0;
-
- public boolean terminate = false;
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper;
- private LineReadWriteUtil lineWriterUtil;
-
- protected HdfsWriterConfiguration hdfsConfiguration;
-
- public WebHdfsPersistWriter() {
- this(new ComponentConfigurator<>(HdfsWriterConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig().getConfig("hdfs")));
- }
-
- public WebHdfsPersistWriter(HdfsWriterConfiguration hdfsConfiguration) {
- this.hdfsConfiguration = hdfsConfiguration;
- this.linesPerFile = hdfsConfiguration.getLinesPerFile().intValue();
- }
-
- /**
- * getURI from hdfsConfiguration.
- * @return URI
- * @throws URISyntaxException URISyntaxException
- */
- // TODO: combine with WebHdfsPersistReader.getURI
- public URI getURI() throws URISyntaxException {
- StringBuilder uriBuilder = new StringBuilder();
- uriBuilder.append(hdfsConfiguration.getScheme());
- uriBuilder.append("://");
- if (StringUtils.isNotEmpty(hdfsConfiguration.getHost())) {
- uriBuilder.append(hdfsConfiguration.getHost() + ":" + hdfsConfiguration.getPort());
- } else {
- uriBuilder.append("/");
- }
- return new URI(uriBuilder.toString());
- }
-
- /**
- * isConnected.
- * @return true if connected, false otherwise
- */
- // TODO: combine with WebHdfsPersistReader.isConnected
- public boolean isConnected() {
- return (client != null);
- }
-
- /**
- * getFileSystem.
- * @return FileSystem
- */
- // TODO: combine with WebHdfsPersistReader.getFileSystem
- public final synchronized FileSystem getFileSystem() {
- // Check to see if we are connected.
- if (!isConnected()) {
- connectToWebHDFS();
- }
- return this.client;
- }
-
- private synchronized void connectToWebHDFS() {
- try {
- LOGGER.info("User : {}", this.hdfsConfiguration.getUser());
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser(this.hdfsConfiguration.getUser());
- ugi.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.SIMPLE);
-
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
- public Void run() throws Exception {
- Configuration conf = new Configuration();
- conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- LOGGER.info("WebURI : {}", getURI().toString());
- client = FileSystem.get(getURI(), conf);
- LOGGER.info("Connected to WebHDFS");
-
- /*
- * ************************************************************************************************
- * This code is an example of how you would work with HDFS and you weren't going over
- * the webHDFS protocol.
- *
- * Smashew: 2013-10-01
- * ************************************************************************************************
- conf.set("fs.defaultFS", "hdfs://hadoop.mdigitallife.com:8020/user/" + userName);
- conf.set("namenode.host","0.0.0.0");
- conf.set("hadoop.job.ugi", userName);
- conf.set(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, "runner");
- fileSystem.createNewFile(new Path("/user/"+ userName + "/test"));
- FileStatus[] status = fs.listStatus(new Path("/user/" + userName));
- for(int i=0;i<status.length;i++)
- {
- LOGGER.info("Directory: {}", status[i].getPath());
- }
- */
-
- return null;
- }
- });
- } catch (Exception ex) {
- LOGGER.error("There was an error connecting to WebHDFS, please check your settings and try again", ex);
- throw new RuntimeException(ex);
- }
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void write(StreamsDatum streamsDatum) {
-
- synchronized (this) {
- // Check to see if we need to reset the file that we are currently working with
- if (this.currentWriter == null || (this.fileLineCounter > this.linesPerFile)) {
- resetFile();
- }
- String line = lineWriterUtil.convertResultToString(streamsDatum);
- writeInternal(line);
- if ( !line.endsWith(this.hdfsConfiguration.getLineDelimiter())) {
- writeInternal(this.hdfsConfiguration.getLineDelimiter());
- }
- int bytesInLine = line.getBytes().length;
-
- totalRecordsWritten++;
- totalByteCount += bytesInLine;
- byteCount += bytesInLine;
-
- if (byteCount > BYTES_BEFORE_FLUSH) {
- try {
- flush();
- } catch (IOException ex) {
- LOGGER.warn("Error flushing to HDFS. Creating a new file and continuing execution. WARNING: There could be data loss.", ex);
- }
- }
- this.fileLineCounter++;
- }
- }
-
- private void writeInternal(String line) {
- try {
- this.currentWriter.write(line);
- } catch (IOException ex) {
- LOGGER.warn("Error writing to HDFS. Attempting to try a new file", ex);
- try {
- resetFile();
- this.currentWriter.write(line);
- } catch (Exception e2) {
- LOGGER.warn("Failed to write even after creating a new file. Attempting to reconnect", e2);
- try {
- connectToWebHDFS();
- resetFile();
- this.currentWriter.write(line);
- } catch (Exception e3) {
- LOGGER.error("Failed to write to HDFS after reconnecting client. Terminating writer.", e3);
- throw new RuntimeException(e3);
- }
- }
-
- }
- }
-
- @Override
- public void flush() throws IOException {
- if (this.currentWriter != null && byteCount > BYTES_BEFORE_FLUSH) {
- this.currentWriter.flush();
- byteCount = 0;
- }
- }
-
- private synchronized void resetFile() {
- // this will keep it thread safe, so we don't create too many files
- if (this.fileLineCounter == 0 && this.currentWriter != null) {
- return;
- }
-
- // Create the path for where the file is going to live.
- Path filePath = this.path.suffix("/" + hdfsConfiguration.getWriterFilePrefix() + "-" + new Date().getTime());
-
- if ( hdfsConfiguration.getCompression().equals(HdfsWriterConfiguration.Compression.GZIP)) {
- filePath = filePath.suffix(".gz");
- } else {
- filePath = filePath.suffix(".tsv");
- }
-
- try {
-
- // if there is a current writer, we must close it first.
- if (this.currentWriter != null) {
- flush();
- close();
- }
-
- this.fileLineCounter = 0;
-
- // Check to see if a file of the same name exists, if it does, then we are not going to be able to proceed.
- if (client.exists(filePath)) {
- throw new RuntimeException("Unable to create file: " + filePath);
- }
-
- if ( hdfsConfiguration.getCompression().equals(HdfsWriterConfiguration.Compression.GZIP)) {
- this.currentWriter = new OutputStreamWriter(new GZIPOutputStream(client.create(filePath)));
- } else {
- this.currentWriter = new OutputStreamWriter(client.create(filePath));
- }
-
- // Add another file to the list of written files.
- writtenFiles.add(filePath);
-
- LOGGER.info("File Created: {}", filePath);
- } catch (Exception ex) {
- LOGGER.error("COULD NOT CreateFile: {}", filePath);
- LOGGER.error(ex.getMessage());
- throw new RuntimeException(ex);
- }
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (this.currentWriter != null) {
- this.currentWriter.flush();
- this.currentWriter.close();
- this.currentWriter = null;
- LOGGER.info("File Closed");
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
- mapper = StreamsJacksonMapper.getInstance();
- lineWriterUtil = LineReadWriteUtil.getInstance(hdfsConfiguration);
- connectToWebHDFS();
- path = new Path(hdfsConfiguration.getPath() + "/" + hdfsConfiguration.getWriterPath());
- }
-
- @Override
- public void cleanUp() {
- try {
- flush();
- } catch (IOException ex) {
- LOGGER.error("Error flushing on cleanup", ex);
- }
- try {
- close();
- } catch (IOException ex) {
- LOGGER.error("Error closing on cleanup", ex);
- }
- }
-
- @Override
- public DatumStatusCounter getDatumStatusCounter() {
- DatumStatusCounter counters = new DatumStatusCounter();
- counters.incrementAttempt(this.totalRecordsWritten);
- counters.incrementStatus(DatumStatus.SUCCESS, this.totalRecordsWritten);
- return counters;
- }
-}
diff --git a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistWriterTask.java b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistWriterTask.java
deleted file mode 100644
index eb808c1..0000000
--- a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistWriterTask.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hdfs;
-
-import org.apache.streams.core.StreamsDatum;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Random;
-
-/**
- * WebHdfsPersistReaderTask writes to hdfs on behalf of
- * @see org.apache.streams.hdfs.WebHdfsPersistWriter
- */
-public class WebHdfsPersistWriterTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(WebHdfsPersistWriterTask.class);
-
- private WebHdfsPersistWriter writer;
-
- public WebHdfsPersistWriterTask(WebHdfsPersistWriter writer) {
- this.writer = writer;
- }
-
- @Override
- public void run() {
-
- while (true) {
- if ( writer.persistQueue.peek() != null ) {
- try {
- StreamsDatum entry = writer.persistQueue.remove();
- writer.write(entry);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- try {
- Thread.sleep(new Random().nextInt(1));
- } catch (InterruptedException e) {}
- }
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-hdfs/src/main/jsonschema/org/apache/streams/hdfs/HdfsConfiguration.json b/streams-contrib/streams-persist-hdfs/src/main/jsonschema/org/apache/streams/hdfs/HdfsConfiguration.json
deleted file mode 100644
index 9a4efd3..0000000
--- a/streams-contrib/streams-persist-hdfs/src/main/jsonschema/org/apache/streams/hdfs/HdfsConfiguration.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.hdfs.HdfsConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {
- "$ref": "../../../../../../../../../streams-components/streams-converters/src/main/jsonschema/org/apache/streams/converter/LineReadWriteConfiguration.json"
- },
- "properties": {
- "scheme": {
- "type": "string",
- "description": "scheme",
- "enum" : ["file", "hdfs", "webhdfs", "s3", "s3n", "s3a", "emrfs"],
- "default": "webhdfs"
- },
- "host": {
- "type": "string",
- "description": "WebHdfs host"
- },
- "port": {
- "type": "integer",
- "description": "WebHdfs port"
- },
- "path": {
- "type": "string",
- "description": "Base path"
- },
- "user": {
- "type": "string",
- "description": "User"
- },
- "password": {
- "type": "string",
- "description": "Password"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-hdfs/src/main/jsonschema/org/apache/streams/hdfs/HdfsReaderConfiguration.json b/streams-contrib/streams-persist-hdfs/src/main/jsonschema/org/apache/streams/hdfs/HdfsReaderConfiguration.json
deleted file mode 100644
index 39f6c8e..0000000
--- a/streams-contrib/streams-persist-hdfs/src/main/jsonschema/org/apache/streams/hdfs/HdfsReaderConfiguration.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.hdfs.HdfsReaderConfiguration",
- "extends": {"$ref":"HdfsConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "readerPath": {
- "type": "string",
- "description": "Path below root path"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-hdfs/src/main/jsonschema/org/apache/streams/hdfs/HdfsWriterConfiguration.json b/streams-contrib/streams-persist-hdfs/src/main/jsonschema/org/apache/streams/hdfs/HdfsWriterConfiguration.json
deleted file mode 100644
index 76d2fca..0000000
--- a/streams-contrib/streams-persist-hdfs/src/main/jsonschema/org/apache/streams/hdfs/HdfsWriterConfiguration.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.hdfs.HdfsWriterConfiguration",
- "extends": {"$ref":"HdfsConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "writerPath": {
- "type": "string",
- "description": "Path "
- },
- "writerFilePrefix": {
- "type": "string",
- "description": "File Prefix"
- },
- "linesPerFile": {
- "type": "integer",
- "description": "Lines Per File",
- "default": 1000
- },
- "compression": {
- "type": "string",
- "description": "compression",
- "enum" : ["none", "gzip"],
- "default": "none"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-hdfs/src/main/resources/components.dot b/streams-contrib/streams-persist-hdfs/src/main/resources/components.dot
deleted file mode 100644
index 6ecaa92..0000000
--- a/streams-contrib/streams-persist-hdfs/src/main/resources/components.dot
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- processors [label="processors", shape="circle"];
-
- subgraph cluster_persisters {
- label="persisters";
- persisters_hdfs_reader [label="WebHdfsPersistReader"]
- persisters_hdfs_writer [label="WebHdfsPersistWriter"]
- }
-
- subgraph cluster_dbs {
- label="dbs";
- hdfs [label="hdfs", shape="tab"]
- }
-
- generators -> providers
- providers -> processors
- processors -> persisters_hdfs_writer [label="StreamsDatum"]
- persisters_hdfs_reader -> processors [label="StreamsDatum"]
- hdfs -> persisters_hdfs_reader
- persisters_hdfs_writer -> hdfs
-
-}
diff --git a/streams-contrib/streams-persist-hdfs/src/main/resources/hdfs-read.conf b/streams-contrib/streams-persist-hdfs/src/main/resources/hdfs-read.conf
deleted file mode 100644
index 2afb688..0000000
--- a/streams-contrib/streams-persist-hdfs/src/main/resources/hdfs-read.conf
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-hdfs: {
- scheme: "hdfs"
- host: "hadoop",
- port: "8020",
- user: "hadoop",
- path: "/user/hadoop",
- readerPath: "inputDir"
-}
diff --git a/streams-contrib/streams-persist-hdfs/src/main/resources/hdfs-write.conf b/streams-contrib/streams-persist-hdfs/src/main/resources/hdfs-write.conf
deleted file mode 100644
index 5c90b89..0000000
--- a/streams-contrib/streams-persist-hdfs/src/main/resources/hdfs-write.conf
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-hdfs: {
- scheme: "hdfs"
- host: "hadoop",
- port: "8020",
- user: "hadoop",
- path: "/user/hadoop",
- writerPath: "outputDir"
-}
diff --git a/streams-contrib/streams-persist-hdfs/src/site/markdown/index.md b/streams-contrib/streams-persist-hdfs/src/site/markdown/index.md
deleted file mode 100644
index 4e97ae6..0000000
--- a/streams-contrib/streams-persist-hdfs/src/site/markdown/index.md
+++ /dev/null
@@ -1,23 +0,0 @@
-streams-persist-hdfs
-=====================
-
-Read and write to HDFS
-
-## Configuration
-
-| Schema |
-|--------|
-| [HdfsConfiguration.json](org/apache/streams/s3/HdfsConfiguration.json "HdfsConfiguration.json") [HdfsConfiguration.html](apidocs/org/apache/streams/s3/HdfsConfiguration.html "javadoc") |
-
-## Components
-
-
-
-| Class | Configuration | Example Configuration(s) |
-|-------|---------------|--------------------------|
-| WebHdfsPersistReader [WebHdfsPersistReader.html](apidocs/org/apache/streams/hdfs/WebHdfsPersistReader.html "javadoc") | [HdfsReaderConfiguration.json](org/apache/streams/hdfs/HdfsReaderConfiguration.json "HdfsReaderConfiguration.json") [HdfsReaderConfiguration.html](apidocs/org/apache/streams/hdfs/HdfsReaderConfiguration.html "javadoc") | [hdfs-read.conf](hdfs-read.conf "hdfs-read.conf") |
-| WebHdfsPersistWriter [WebHdfsPersistWriter.html](apidocs/org/apache/streams/hdfs/WebHdfsPersistWriter "javadoc") | [HdfsWriterConfiguration.json](org/apache/streams/hdfs/HdfsWriterConfiguration.json "HdfsWriterConfiguration.json") [HdfsWriterConfiguration.html](apidocs/org/apache/streams/hdfs/HdfsWriterConfiguration.html "javadoc") | [hdfs-write.conf](hdfs-write.conf "hdfs-write.conf") |
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/HdfsPersistConfigTest.java b/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/HdfsPersistConfigTest.java
deleted file mode 100644
index 6d7aaf6..0000000
--- a/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/HdfsPersistConfigTest.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hdfs.test;
-
-import org.apache.streams.hdfs.HdfsConfiguration;
-import org.apache.streams.hdfs.HdfsReaderConfiguration;
-import org.apache.streams.hdfs.HdfsWriterConfiguration;
-import org.apache.streams.hdfs.WebHdfsPersistReader;
-import org.apache.streams.hdfs.WebHdfsPersistWriter;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URISyntaxException;
-
-/**
- * Test for checking that strings append to FS paths as expected
- */
-public class HdfsPersistConfigTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(HdfsPersistConfigTest.class);
-
- @Test
- public void getWriterFileUriTest() {
- HdfsWriterConfiguration writerConfiguration = new HdfsWriterConfiguration();
- writerConfiguration.setScheme(HdfsConfiguration.Scheme.FILE);
- writerConfiguration.setPath("path");
- writerConfiguration.setWriterPath("writerPath");
- writerConfiguration.setUser("cloudera");
-
- WebHdfsPersistWriter webHdfsPersistWriter = new WebHdfsPersistWriter(writerConfiguration);
-
- String uri = null;
- try {
- uri = webHdfsPersistWriter.getURI().toString();
- } catch (URISyntaxException e) {
- Assert.fail("URI Syntax");
- }
- Assert.assertArrayEquals(uri.toCharArray(), ("file:///").toCharArray());
- webHdfsPersistWriter.prepare(null);
- Assert.assertTrue(webHdfsPersistWriter.isConnected());
- }
-
- @Test
- public void getWriterHdfsUriTest() {
- HdfsWriterConfiguration writerConfiguration = new HdfsWriterConfiguration();
- writerConfiguration.setScheme(HdfsConfiguration.Scheme.HDFS);
- writerConfiguration.setHost("localhost");
- writerConfiguration.setPort(9000L);
- writerConfiguration.setPath("path");
- writerConfiguration.setWriterPath("writerPath");
- writerConfiguration.setUser("cloudera");
-
- WebHdfsPersistWriter webHdfsPersistWriter = new WebHdfsPersistWriter(writerConfiguration);
-
- String uri = null;
- try {
- uri = webHdfsPersistWriter.getURI().toString();
- } catch (URISyntaxException e) {
- Assert.fail("URI Syntax");
- }
- Assert.assertArrayEquals(uri.toCharArray(), ("hdfs://localhost:9000").toCharArray());
-
- }
-
- @Test
- public void getWriterWebHdfsUriTest() {
- HdfsWriterConfiguration writerConfiguration = new HdfsWriterConfiguration();
- writerConfiguration.setScheme(HdfsConfiguration.Scheme.WEBHDFS);
- writerConfiguration.setHost("localhost");
- writerConfiguration.setPort(57000L);
- writerConfiguration.setPath("path");
- writerConfiguration.setWriterPath("writerPath");
- writerConfiguration.setUser("cloudera");
-
- WebHdfsPersistWriter webHdfsPersistWriter = new WebHdfsPersistWriter(writerConfiguration);
-
- String uri = null;
- try {
- uri = webHdfsPersistWriter.getURI().toString();
- } catch (URISyntaxException e) {
- Assert.fail("URI Syntax");
- }
- Assert.assertArrayEquals(uri.toCharArray(), ("webhdfs://localhost:57000").toCharArray());
-
- }
-
- @Test
- public void getReaderFileUriTest() {
- HdfsReaderConfiguration readerConfiguration = new HdfsReaderConfiguration();
- readerConfiguration.setScheme(HdfsConfiguration.Scheme.FILE);
- readerConfiguration.setPath("path");
- readerConfiguration.setReaderPath("readerPath");
-
- WebHdfsPersistReader webHdfsPersistReader = new WebHdfsPersistReader(readerConfiguration);
-
- String uri = null;
- try {
- uri = webHdfsPersistReader.getURI().toString();
- } catch (URISyntaxException e) {
- Assert.fail("URI Syntax");
- }
- Assert.assertArrayEquals(uri.toCharArray(), ("file:///").toCharArray());
- }
-
- @Test
- public void getReaderHdfsUriTest() {
- HdfsReaderConfiguration readerConfiguration = new HdfsReaderConfiguration();
- readerConfiguration.setScheme(HdfsConfiguration.Scheme.HDFS);
- readerConfiguration.setHost("localhost");
- readerConfiguration.setPort(9000L);
- readerConfiguration.setPath("path");
- readerConfiguration.setReaderPath("readerPath");
-
- WebHdfsPersistReader webHdfsPersistReader = new WebHdfsPersistReader(readerConfiguration);
-
- String uri = null;
- try {
- uri = webHdfsPersistReader.getURI().toString();
- } catch (URISyntaxException e) {
- Assert.fail("URI Syntax");
- }
- Assert.assertArrayEquals(uri.toCharArray(), ("hdfs://localhost:9000").toCharArray());
-
- }
-
- @Test
- public void getReaderWebHdfsUriTest() {
- HdfsReaderConfiguration readerConfiguration = new HdfsReaderConfiguration();
- readerConfiguration.setScheme(HdfsConfiguration.Scheme.WEBHDFS);
- readerConfiguration.setHost("localhost");
- readerConfiguration.setPort(57000L);
- readerConfiguration.setPath("path");
- readerConfiguration.setReaderPath("readerPath");
-
- WebHdfsPersistReader webHdfsPersistReader = new WebHdfsPersistReader(readerConfiguration);
-
- String uri = null;
- try {
- uri = webHdfsPersistReader.getURI().toString();
- } catch (URISyntaxException e) {
- Assert.fail("URI Syntax");
- }
- Assert.assertArrayEquals(uri.toCharArray(), ("webhdfs://localhost:57000").toCharArray());
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/TestHdfsPersist.java b/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/TestHdfsPersist.java
deleted file mode 100644
index 798a8b8..0000000
--- a/streams-contrib/streams-persist-hdfs/src/test/java/org/apache/streams/hdfs/test/TestHdfsPersist.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.hdfs.test;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.hdfs.HdfsConfiguration;
-import org.apache.streams.hdfs.HdfsReaderConfiguration;
-import org.apache.streams.hdfs.HdfsWriterConfiguration;
-import org.apache.streams.hdfs.WebHdfsPersistReader;
-import org.apache.streams.hdfs.WebHdfsPersistWriter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.io.IOUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Test reading and writing documents
- */
-public class TestHdfsPersist {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TestHdfsPersist.class);
-
- private ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- @Before
- public void setup() {
- File file = new File("/target/TestHdfsPersist/");
- if (file.exists()) {
- file.delete();
- }
- }
-
- @Test
- public void TestHdfsPersist() throws Exception {
-
- List<List<String>> fieldArrays = new ArrayList<>();
- fieldArrays.add(new ArrayList<>());
- fieldArrays.add(Collections.singletonList("ID"));
- fieldArrays.add(Arrays.asList("ID", "DOC"));
- fieldArrays.add(Arrays.asList("ID", "TS", "DOC"));
- fieldArrays.add(Arrays.asList("ID", "TS", "META", "DOC"));
-
- for (List<String> fields : fieldArrays) {
- TestHdfsPersistCase(fields);
- }
-
- }
-
- public void TestHdfsPersistCase(List<String> fields) throws Exception {
-
- HdfsConfiguration hdfsConfiguration = new HdfsConfiguration().withScheme(HdfsConfiguration.Scheme.FILE).withHost("localhost").withUser("cloudera").withPath("target/TestHdfsPersist");
- hdfsConfiguration.setFields(fields);
- HdfsWriterConfiguration hdfsWriterConfiguration = MAPPER.convertValue(hdfsConfiguration, HdfsWriterConfiguration.class);
- if (fields.size() % 2 == 1) {
- hdfsWriterConfiguration.setCompression(HdfsWriterConfiguration.Compression.GZIP);
- }
- hdfsWriterConfiguration.setWriterFilePrefix("activities");
- hdfsWriterConfiguration.setWriterPath(Integer.toString(fields.size()));
- WebHdfsPersistWriter writer = new WebHdfsPersistWriter(hdfsWriterConfiguration);
-
- writer.prepare(null);
-
- InputStream testActivityFolderStream = TestHdfsPersist.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- int count = 0;
-
- for (String file : files) {
- LOGGER.info("File: " + file);
- InputStream testActivityFileStream = TestHdfsPersist.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- activity.getAdditionalProperties().remove("$license");
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- writer.write(datum);
- LOGGER.info("Wrote: " + activity.getVerb());
- count++;
- }
-
- writer.cleanUp();
-
- HdfsReaderConfiguration hdfsReaderConfiguration = MAPPER.convertValue(hdfsConfiguration, HdfsReaderConfiguration.class);
-
- WebHdfsPersistReader reader = new WebHdfsPersistReader(hdfsReaderConfiguration);
- hdfsReaderConfiguration.setReaderPath(Integer.toString(fields.size()));
-
- reader.prepare(null);
-
- StreamsResultSet resultSet = reader.readAll();
-
- Assert.assertEquals(resultSet.size(), count);
-
- }
-}
diff --git a/streams-contrib/streams-persist-hdfs/src/test/resources/hdfs-read.conf b/streams-contrib/streams-persist-hdfs/src/test/resources/hdfs-read.conf
deleted file mode 100644
index 5b806f4..0000000
--- a/streams-contrib/streams-persist-hdfs/src/test/resources/hdfs-read.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-hdfs {
- host = "localhost"
- port = "50070"
- path = "/user/cloudera"
- user = "cloudera"
- password = "cloudera"
- writerPath = "/history/twitter/example"
- writerFilePrefix = "streams-"
-}
diff --git a/streams-contrib/streams-persist-hdfs/src/test/resources/hdfs-write.conf b/streams-contrib/streams-persist-hdfs/src/test/resources/hdfs-write.conf
deleted file mode 100644
index b375fda..0000000
--- a/streams-contrib/streams-persist-hdfs/src/test/resources/hdfs-write.conf
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-hdfs {
- host = "localhost"
- port = "50070"
- path = "/user/cloudera"
- user = "cloudera"
- password = "cloudera"
-}
diff --git a/streams-contrib/streams-persist-kafka/README.md b/streams-contrib/streams-persist-kafka/README.md
deleted file mode 100644
index 10eb96a..0000000
--- a/streams-contrib/streams-persist-kafka/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-persist-kafka
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-persist-kafka/pom.xml b/streams-contrib/streams-persist-kafka/pom.xml
deleted file mode 100644
index 1059bee..0000000
--- a/streams-contrib/streams-persist-kafka/pom.xml
+++ /dev/null
@@ -1,164 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-kafka</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Kafka Module</description>
-
- <properties>
- <scala.version>2.9.2</scala.version>
- <kafka.version>0.8.0</kafka.version>
- <clojure.version>1.2.0</clojure.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_${scala.version}</artifactId>
- <version>${kafka.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jdmk</groupId>
- <artifactId>jmxtools</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jmx</groupId>
- <artifactId>jmxri</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.clojure</groupId>
- <artifactId>clojure</artifactId>
- <version>${clojure.version}</version>
- </dependency>
- <dependency>
- <groupId>org.clojure</groupId>
- <artifactId>clojure-contrib</artifactId>
- <version>${clojure.version}</version>
- </dependency>
- <dependency>
- <groupId>com.101tec</groupId>
- <artifactId>zkclient</artifactId>
- <version>0.3</version>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.kafka.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistReader.java b/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistReader.java
deleted file mode 100644
index edca3de..0000000
--- a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistReader.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.kafka;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import kafka.consumer.Consumer;
-import kafka.consumer.ConsumerConfig;
-import kafka.consumer.KafkaStream;
-import kafka.consumer.Whitelist;
-import kafka.javaapi.consumer.ConsumerConnector;
-import kafka.serializer.StringDecoder;
-import kafka.utils.VerifiableProperties;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.List;
-import java.util.Properties;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-/**
- * KafkaPersistReader reads documents from kafka.
- */
-public class KafkaPersistReader implements StreamsPersistReader, Serializable {
-
- public static final String STREAMS_ID = "KafkaPersistReader";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(KafkaPersistReader.class);
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = new ObjectMapper();
-
- private KafkaConfiguration config;
-
- private ConsumerConnector consumerConnector;
-
- public List<KafkaStream<String, String>> inStreams;
-
- private ExecutorService executor = Executors.newSingleThreadExecutor();
-
- /**
- * KafkaPersistReader constructor - resolves KafkaConfiguration from JVM 'kafka'.
- */
- public KafkaPersistReader() {
- this.config = new ComponentConfigurator<>(KafkaConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("kafka"));
- this.persistQueue = new ConcurrentLinkedQueue<>();
- }
-
- /**
- * KafkaPersistReader constructor - uses supplied persistQueue.
- */
- public KafkaPersistReader(Queue<StreamsDatum> persistQueue) {
- this.config = new ComponentConfigurator<>(KafkaConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("kafka"));
- this.persistQueue = persistQueue;
- }
-
- public void setConfig(KafkaConfiguration config) {
- this.config = config;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
-
- Properties props = new Properties();
- props.setProperty("serializer.encoding", "UTF8");
-
- ConsumerConfig consumerConfig = new ConsumerConfig(props);
-
- consumerConnector = Consumer.createJavaConsumerConnector(consumerConfig);
-
- Whitelist topics = new Whitelist(config.getTopic());
- VerifiableProperties vprops = new VerifiableProperties(props);
-
- inStreams = consumerConnector.createMessageStreamsByFilter(topics, 1, new StringDecoder(vprops), new StringDecoder(vprops));
-
- for (final KafkaStream stream : inStreams) {
- executor.submit(new KafkaPersistReaderTask(this, stream));
- }
-
- }
-
- @Override
- public StreamsResultSet readAll() {
- return readCurrent();
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- return null;
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger bigInteger) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime dateTime, DateTime dateTime2) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return !executor.isShutdown() && !executor.isTerminated();
- }
-
- private static ConsumerConfig createConsumerConfig(String zookeeper, String groupId) {
- Properties props = new Properties();
- props.put("zookeeper.connect", zookeeper);
- props.put("group.id", groupId);
- props.put("zookeeper.session.timeout.ms", "400");
- props.put("zookeeper.sync.time.ms", "200");
- props.put("auto.commit.interval.ms", "1000");
- return new ConsumerConfig(props);
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
- consumerConnector.shutdown();
- while ( !executor.isTerminated()) {
- try {
- executor.awaitTermination(5, TimeUnit.SECONDS);
- } catch (InterruptedException interrupt) {
- LOGGER.trace("Interrupt", interrupt);
- }
- }
- }
-}
diff --git a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistReaderTask.java b/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistReaderTask.java
deleted file mode 100644
index a36246c..0000000
--- a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistReaderTask.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.kafka;
-
-import org.apache.streams.core.StreamsDatum;
-
-import kafka.consumer.KafkaStream;
-import kafka.message.MessageAndMetadata;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Random;
-
-/**
- * KafkaPersistReaderTask reads documents from kafka on behalf of
- * @see org.apache.streams.kafka.KafkaPersistReader
- */
-public class KafkaPersistReaderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(KafkaPersistReaderTask.class);
-
- private KafkaPersistReader reader;
- private KafkaStream<String,String> stream;
-
- public KafkaPersistReaderTask(KafkaPersistReader reader, KafkaStream<String,String> stream) {
- this.reader = reader;
- this.stream = stream;
- }
-
- @Override
- public void run() {
-
- MessageAndMetadata<String,String> item;
- while (true) {
-
- for (MessageAndMetadata<String, String> aStream : stream) {
- item = aStream;
- reader.persistQueue.add(new StreamsDatum(item.message()));
- }
- try {
- Thread.sleep(new Random().nextInt(100));
- } catch (InterruptedException interrupt) {
- LOGGER.trace("Interrupt", interrupt);
- }
- }
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriter.java b/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriter.java
deleted file mode 100644
index 9f696c7..0000000
--- a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriter.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.kafka;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.util.GuidUtils;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import kafka.javaapi.producer.Producer;
-import kafka.producer.KeyedMessage;
-import kafka.producer.ProducerConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.util.Properties;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-/**
- * KafkaPersistWriter writes documents to kafka.
- */
-public class KafkaPersistWriter implements StreamsPersistWriter, Serializable, Runnable {
-
- public static final String STREAMS_ID = "KafkaPersistWriter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(KafkaPersistWriter.class);
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = new ObjectMapper();
-
- private KafkaConfiguration config;
-
- private Producer<String, String> producer;
-
- /**
- * KafkaPersistWriter constructor - resolves KafkaConfiguration from JVM 'kafka'.
- */
- public KafkaPersistWriter() {
- this.config = new ComponentConfigurator<>(KafkaConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("kafka"));
- this.persistQueue = new ConcurrentLinkedQueue<>();
- }
-
- /**
- * KafkaPersistWriter constructor - uses supplied persistQueue.
- */
- public KafkaPersistWriter(Queue<StreamsDatum> persistQueue) {
- this.config = new ComponentConfigurator<>(KafkaConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("kafka"));
- this.persistQueue = persistQueue;
- }
-
- public void setConfig(KafkaConfiguration config) {
- this.config = config;
- }
-
- /**
- * run persist writer thread
- */
- public void start() {
- Properties props = new Properties();
-
- props.put("metadata.broker.list", config.getBrokerlist());
- props.put("serializer.class", "kafka.serializer.StringEncoder");
- props.put("partitioner.class", "org.apache.streams.kafka.StreamsPartitioner");
- props.put("request.required.acks", "1");
-
- ProducerConfig config = new ProducerConfig(props);
-
- producer = new Producer<>(config);
-
- new Thread(new KafkaPersistWriterTask(this)).start();
- }
-
- public void stop() {
- producer.close();
- }
-
- public void setPersistQueue(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
- }
-
- public Queue<StreamsDatum> getPersistQueue() {
- return this.persistQueue;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void write(StreamsDatum entry) {
-
- try {
-
- String text = mapper.writeValueAsString(entry);
-
- String hash = GuidUtils.generateGuid(text);
-
- KeyedMessage<String, String> data = new KeyedMessage<>(config.getTopic(), hash, text);
-
- producer.send(data);
-
- } catch (JsonProcessingException ex) {
- LOGGER.warn("save: {}", ex);
- }
- }
-
- @Override
- public void run() {
- start();
- }
-
- @Override
- public void prepare(Object configurationObject) {
- start();
- }
-
- @Override
- public void cleanUp() {
- stop();
- }
-}
diff --git a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriterTask.java b/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriterTask.java
deleted file mode 100644
index dae7aa2..0000000
--- a/streams-contrib/streams-persist-kafka/src/main/java/org/apache/streams/kafka/KafkaPersistWriterTask.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.kafka;
-
-import org.apache.streams.core.StreamsDatum;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Random;
-
-/**
- * KafkaPersistWriterTask writes documents to kafka on behalf of
- * @see org.apache.streams.kafka.KafkaPersistWriter
- */
-public class KafkaPersistWriterTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(KafkaPersistWriterTask.class);
-
- private KafkaPersistWriter writer;
-
- public KafkaPersistWriterTask(KafkaPersistWriter writer) {
- this.writer = writer;
- }
-
- @Override
- public void run() {
-
- while (true) {
- if ( writer.getPersistQueue().peek() != null ) {
- try {
- StreamsDatum entry = writer.persistQueue.remove();
- writer.write(entry);
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
- try {
- Thread.sleep(new Random().nextInt(100));
- } catch (InterruptedException interrupt) {
- LOGGER.trace("Interrupt", interrupt);
- }
- }
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-kafka/src/main/jsonschema/org/apache/streams/kafka/KafkaConfiguration.json b/streams-contrib/streams-persist-kafka/src/main/jsonschema/org/apache/streams/kafka/KafkaConfiguration.json
deleted file mode 100644
index 88e5554..0000000
--- a/streams-contrib/streams-persist-kafka/src/main/jsonschema/org/apache/streams/kafka/KafkaConfiguration.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.kafka.KafkaConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "brokerlist": {
- "type": "string",
- "description": "A comma-delimited list of broker nodes, used by producer"
- },
- "zkconnect": {
- "type": "string",
- "description": "A comma-delimited list of zookeeper host:ports, used by consumer"
- },
- "topic": {
- "type": "string",
- "description": "A topic to read/write from"
- },
- "groupId": {
- "type": "string",
- "description": "A required field for partitioning distributed consumers"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-kafka/src/main/resources/components.dot b/streams-contrib/streams-persist-kafka/src/main/resources/components.dot
deleted file mode 100644
index 46d4559..0000000
--- a/streams-contrib/streams-persist-kafka/src/main/resources/components.dot
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- processors [label="processors", shape="circle"];
-
- subgraph cluster_persisters {
- label="persisters";
- persisters_kafka_reader [label="KafkaPersistReader"]
- persisters_kafka_writer [label="KafkaPersistWriter"]
- }
-
- subgraph cluster_queues {
- label="queues";
- kafka [label="kafka", shape="trapezium"]
- }
-
- generators -> providers
- providers -> processors
- processors -> persisters_kafka_writer [label="StreamsDatum"]
- persisters_kafka_reader -> processors [label="StreamsDatum[String]"]
- persisters_kafka_writer -> kafka
- kafka -> persisters_kafka_reader
-}
diff --git a/streams-contrib/streams-persist-kafka/src/main/resources/kafka.conf b/streams-contrib/streams-persist-kafka/src/main/resources/kafka.conf
deleted file mode 100644
index a75a948..0000000
--- a/streams-contrib/streams-persist-kafka/src/main/resources/kafka.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-kafka {
- brokerlist = "localhost:9092"
- zkconnect = "localhost:2181"
- topic = "topic"
- groupId = "group"
-}
diff --git a/streams-contrib/streams-persist-kafka/src/main/resources/reference.properties b/streams-contrib/streams-persist-kafka/src/main/resources/reference.properties
deleted file mode 100644
index 967264d..0000000
--- a/streams-contrib/streams-persist-kafka/src/main/resources/reference.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-############################# Producer Basics #############################
-
-# list of brokers used for bootstrapping knowledge about the rest of the cluster
-# format: host1:port1,host2:port2 ...
-kafka.metadata.broker.list=localhost:9092
-
-# name of the partitioner class for partitioning events; default partition spreads data randomly
-#partitioner.class=
-
-# specifies whether the messages are sent asynchronously (async) or synchronously (sync)
-kafka.producer.type=sync
-
-# specify the compression codec for all data generated: none , gzip, snappy.
-# the old config values work as well: 0, 1, 2 for none, gzip, snappy, respectivally
-kafka.compression.codec=none
-
-# message encoder
-kafka.serializer.class=kafka.serializer.DefaultEncoder
-
-# allow topic level compression
-#compressed.topics=
-
-############################# Async Producer #############################
-# maximum time, in milliseconds, for buffering data on the producer queue
-#queue.buffering.max.ms=
-
-# the maximum size of the blocking queue for buffering on the producer
-#queue.buffering.max.messages=
-
-# Timeout for event enqueue:
-# 0: events will be enqueued immediately or dropped if the queue is full
-# -ve: enqueue will block indefinitely if the queue is full
-# +ve: enqueue will block up to this many milliseconds if the queue is full
-#queue.enqueue.timeout.ms=
-
-# the number of messages batched at the producer
-#batch.num.messages=
-
-kafka.groupid=kafka
-
-kafka.zk.connect=localhost:2181
-
-
diff --git a/streams-contrib/streams-persist-kafka/src/site/markdown/index.md b/streams-contrib/streams-persist-kafka/src/site/markdown/index.md
deleted file mode 100644
index 35cc68b..0000000
--- a/streams-contrib/streams-persist-kafka/src/site/markdown/index.md
+++ /dev/null
@@ -1,23 +0,0 @@
-streams-persist-kafka
-=====================
-
-Read/write to/from Kafka
-
-## Configuration
-
-| Schema |
-|--------|
-| [KafkaConfiguration.json](org/apache/streams/kafka/KafkaConfiguration.json "KafkaConfiguration.json") [KafkaConfiguration.html](apidocs/org/apache/streams/kafka/KafkaConfiguration.html "javadoc") |
-
-## Components
-
-
-
-| Class |
-|-------|
-| KafkaPersistReader [KafkaPersistReader.html](apidocs/org/apache/streams/kafka/KafkaPersistReader.html "javadoc")
-| KafkaPersistWriter [KafkaPersistWriter.html](apidocs/org/apache/streams/kafka/KafkaPersistWriter.html "javadoc")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-mongo/README.md b/streams-contrib/streams-persist-mongo/README.md
deleted file mode 100644
index 7d73505..0000000
--- a/streams-contrib/streams-persist-mongo/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-persist-mongo
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-persist-mongo/pom.xml b/streams-contrib/streams-persist-mongo/pom.xml
deleted file mode 100644
index 1a816d9..0000000
--- a/streams-contrib/streams-persist-mongo/pom.xml
+++ /dev/null
@@ -1,219 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-mongo</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Mongo Module</description>
-
- <properties>
- <mongo-driver.version>3.5.0</mongo-driver.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.mongodb</groupId>
- <artifactId>mongo-java-driver</artifactId>
- <version>${mongo-driver.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.mongo.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>resource-dependencies</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <version>${failsafe.plugin.version}</version>
- <configuration>
- <!-- Run integration test suite rather than individual tests. -->
- <excludes>
- <exclude>**/*Test.java</exclude>
- <exclude>**/*Tests.java</exclude>
- </excludes>
- <includes>
- <include>**/*IT.java</include>
- </includes>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <includes>
- <include>**/*.conf</include>
- <include>**/*.json</include>
- <include>**/*.class</include>
- </includes>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <id>dockerITs</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>skipITs</name>
- <value>false</value>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>io.fabric8</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- <configuration combine.self="override">
- <watchInterval>500</watchInterval>
- <logDate>default</logDate>
- <verbose>true</verbose>
- <autoPull>on</autoPull>
- <images>
- <image>
- <name>mongo:3.2.0</name>
- <alias>mongo</alias>
- <run>
- <namingStrategy>none</namingStrategy>
- <ports>
- <port>${mongo.tcp.host}:${mongo.tcp.port}:27017</port>
- </ports>
- <portPropertyFile>mongo.properties</portPropertyFile>
- <log>
- <enabled>true</enabled>
- <date>default</date>
- <color>cyan</color>
- </log>
- </run>
- <watch>
- <mode>none</mode>
- </watch>
- </image>
- </images>
- </configuration>
-
- </plugin>
-
- </plugins>
- </build>
-
- </profile>
- </profiles>
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistReader.java b/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistReader.java
deleted file mode 100644
index 2fdd821..0000000
--- a/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistReader.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.mongo;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Queues;
-import com.mongodb.DB;
-import com.mongodb.DBCollection;
-import com.mongodb.DBCursor;
-import com.mongodb.DBObject;
-import com.mongodb.MongoClient;
-import com.mongodb.MongoCredential;
-import com.mongodb.ServerAddress;
-import org.apache.commons.lang3.StringUtils;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.math.BigInteger;
-import java.util.Queue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * MongoPersistReader reads documents from mongo.
- */
-public class MongoPersistReader implements StreamsPersistReader {
-
- public static final String STREAMS_ID = "MongoPersistReader";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MongoPersistReader.class);
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- private volatile AtomicLong lastWrite = new AtomicLong(System.currentTimeMillis());
-
- private ExecutorService executor;
-
- private MongoConfiguration config;
-
- protected MongoClient client;
- protected DB db;
- protected DBCollection collection;
-
- protected DBCursor cursor;
-
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- /**
- * KafkaPersistReader constructor - resolves KafkaConfiguration from JVM 'mongo'.
- */
- public MongoPersistReader() {
- this.config = new ComponentConfigurator<>(MongoConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("mongo"));
- }
-
- /**
- * KafkaPersistReader constructor - uses supplied MongoConfiguration.
- * @param config config
- */
- public MongoPersistReader(MongoConfiguration config) {
- this.config = config;
- }
-
- /**
- * KafkaPersistReader constructor - uses supplied persistQueue.
- * @param persistQueue persistQueue
- */
- public MongoPersistReader(Queue<StreamsDatum> persistQueue) {
- this.config = new ComponentConfigurator<>(MongoConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("mongo"));
- this.persistQueue = persistQueue;
- }
-
- public void setPersistQueue(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
- }
-
- public Queue<StreamsDatum> getPersistQueue() {
- return persistQueue;
- }
-
- public void stop() {
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- connectToMongo();
-
- if ( client == null
- || collection == null ) {
- throw new RuntimeException("Unable to connect!");
- }
- cursor = collection.find();
-
- if ( cursor == null
- || !cursor.hasNext()) {
- throw new RuntimeException("Collection not present or empty!");
- }
-
- persistQueue = constructQueue();
-
- executor = Executors.newSingleThreadExecutor();
-
- }
-
- @Override
- public void cleanUp() {
- stop();
- }
-
- protected StreamsDatum prepareDatum(DBObject dbObject) {
-
- ObjectNode objectNode;
- String id;
-
- try {
- objectNode = mapper.readValue(dbObject.toString(), ObjectNode.class);
- id = objectNode.get("_id").get("$oid").asText();
- objectNode.remove("_id");
- } catch (IOException ex) {
- LOGGER.warn("document isn't valid JSON.");
- return null;
- }
-
- return new StreamsDatum(objectNode, id);
- }
-
- private synchronized void connectToMongo() {
-
- ServerAddress serverAddress = new ServerAddress(config.getHost(), config.getPort().intValue());
-
- if (StringUtils.isNotEmpty(config.getUser()) && StringUtils.isNotEmpty(config.getPassword())) {
- MongoCredential credential =
- MongoCredential.createCredential(config.getUser(), config.getDb(), config.getPassword().toCharArray());
- client = new MongoClient(serverAddress, Stream.of(credential).collect(Collectors.toList()));
- } else {
- client = new MongoClient(serverAddress);
- }
-
- db = client.getDB(config.getDb());
-
- if (!db.collectionExists(config.getCollection())) {
- db.createCollection(config.getCollection(), null);
- }
-
- collection = db.getCollection(config.getCollection());
- }
-
- @Override
- public StreamsResultSet readAll() {
-
- try (DBCursor cursor = collection.find()) {
- while (cursor.hasNext()) {
- DBObject dbObject = cursor.next();
- StreamsDatum datum = prepareDatum(dbObject);
- write(datum);
- }
- }
-
- return readCurrent();
- }
-
- @Override
- public void startStream() {
-
- LOGGER.debug("startStream");
- MongoPersistReaderTask readerTask = new MongoPersistReaderTask(this);
- Thread readerTaskThread = new Thread(readerTask);
- Future future = executor.submit(readerTaskThread);
-
- while ( !future.isDone() && !future.isCancelled()) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException interrupt) {
- LOGGER.trace("Interrupt", interrupt);
- }
- }
-
- executor.shutdown();
-
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- try {
- lock.writeLock().lock();
- current = new StreamsResultSet(persistQueue);
- current.setCounter(new DatumStatusCounter());
- persistQueue = constructQueue();
- } finally {
- lock.writeLock().unlock();
- }
-
- return current;
- }
-
- //The locking may appear to be counter intuitive but we really don't care if multiple threads offer to the queue
- //as it is a synchronized queue. What we do care about is that we don't want to be offering to the current reference
- //if the queue is being replaced with a new instance
- protected void write(StreamsDatum entry) {
- boolean success;
- do {
- try {
- lock.readLock().lock();
- success = persistQueue.offer(entry);
- Thread.yield();
- } finally {
- lock.readLock().unlock();
- }
- }
- while (!success);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return !executor.isTerminated() || !executor.isShutdown();
- }
-
- private Queue<StreamsDatum> constructQueue() {
- return Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
- }
-
- public class MongoPersistReaderTask implements Runnable {
-
- private MongoPersistReader reader;
-
- public MongoPersistReaderTask(MongoPersistReader reader) {
- this.reader = reader;
- }
-
- @Override
- public void run() {
-
- try {
- while (reader.cursor.hasNext()) {
- DBObject dbObject = reader.cursor.next();
- StreamsDatum datum = reader.prepareDatum(dbObject);
- reader.write(datum);
- }
- } finally {
- reader.cursor.close();
- }
-
- }
-
- }
-}
diff --git a/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistWriter.java b/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistWriter.java
deleted file mode 100644
index dd3faa2..0000000
--- a/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistWriter.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.mongo;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.mongodb.DB;
-import com.mongodb.DBCollection;
-import com.mongodb.DBObject;
-import com.mongodb.MongoClient;
-import com.mongodb.MongoCredential;
-import com.mongodb.ServerAddress;
-import com.mongodb.util.JSON;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.Flushable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Queue;
-import java.util.Random;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public class MongoPersistWriter implements StreamsPersistWriter, Runnable, Flushable, Closeable {
-
- public static final String STREAMS_ID = "MongoPersistWriter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MongoPersistWriter.class);
-
- private static final long MAX_WRITE_LATENCY = 1000;
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- private volatile AtomicLong lastWrite = new AtomicLong(System.currentTimeMillis());
- private ScheduledExecutorService backgroundFlushTask = Executors.newSingleThreadScheduledExecutor();
-
- private MongoConfiguration config;
-
- protected MongoClient client;
- protected DB db;
- protected DBCollection collection;
-
- protected List<DBObject> insertBatch = new ArrayList<>();
-
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- public MongoPersistWriter() {
- this(new ComponentConfigurator<>(MongoConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("mongo")));
- }
-
- public MongoPersistWriter(MongoConfiguration config) {
- this.config = config;
- }
-
- public void setPersistQueue(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
- }
-
- public Queue<StreamsDatum> getPersistQueue() {
- return persistQueue;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void write(StreamsDatum streamsDatum) {
-
- DBObject dbObject = prepareObject(streamsDatum);
- if (dbObject != null) {
- addToBatch(dbObject);
- flushIfNecessary();
- }
- }
-
- @Override
- public void flush() throws IOException {
- try {
- LOGGER.debug("Attempting to flush {} items to mongo", insertBatch.size());
- lock.writeLock().lock();
- collection.insert(insertBatch);
- lastWrite.set(System.currentTimeMillis());
- insertBatch = new ArrayList<>();
- } finally {
- lock.writeLock().unlock();
- }
-
- }
-
- public synchronized void close() throws IOException {
- client.close();
- backgroundFlushTask.shutdownNow();
- }
-
- /**
- * start write thread.
- */
- public void start() {
- connectToMongo();
- backgroundFlushTask.scheduleAtFixedRate(this::flushIfNecessary, 0, MAX_WRITE_LATENCY * 2, TimeUnit.MILLISECONDS);
- }
-
- /**
- * stop.
- */
- public void stop() {
-
- try {
- flush();
- } catch (IOException ex) {
- LOGGER.error("Error flushing", ex);
- }
- try {
- close();
- } catch (IOException ex) {
- LOGGER.error("Error closing", ex);
- }
- try {
- backgroundFlushTask.shutdown();
- // Wait a while for existing tasks to terminate
- if (!backgroundFlushTask.awaitTermination(15, TimeUnit.SECONDS)) {
- backgroundFlushTask.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!backgroundFlushTask.awaitTermination(15, TimeUnit.SECONDS)) {
- LOGGER.error("Stream did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- backgroundFlushTask.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
-
- }
-
- @Override
- public void run() {
-
- while (true) {
- if (persistQueue.peek() != null) {
- try {
- StreamsDatum entry = persistQueue.remove();
- write(entry);
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
- try {
- Thread.sleep(new Random().nextInt(1));
- } catch (InterruptedException interrupt) {
- LOGGER.trace("Interrupt", interrupt);
- }
- }
-
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.persistQueue = new ConcurrentLinkedQueue<>();
- start();
- }
-
- @Override
- public void cleanUp() {
- stop();
- }
-
- protected void flushIfNecessary() {
- long lastLatency = System.currentTimeMillis() - lastWrite.get();
- //Flush iff the size > 0 AND the size is divisible by 100 or the time between now and the last flush is greater
- //than the maximum desired latency
- if (insertBatch.size() > 0 && (insertBatch.size() % 100 == 0 || lastLatency > MAX_WRITE_LATENCY)) {
- try {
- flush();
- } catch (IOException ex) {
- LOGGER.error("Error writing to Mongo", ex);
- }
- }
- }
-
- protected void addToBatch(DBObject dbObject) {
- try {
- lock.readLock().lock();
- insertBatch.add(dbObject);
- } finally {
- lock.readLock().unlock();
- }
- }
-
- protected DBObject prepareObject(StreamsDatum streamsDatum) {
- DBObject dbObject = null;
- if (streamsDatum.getDocument() instanceof String) {
- dbObject = (DBObject) JSON.parse((String) streamsDatum.getDocument());
- } else {
- try {
- ObjectNode node = mapper.valueToTree(streamsDatum.getDocument());
- dbObject = (DBObject) JSON.parse(node.toString());
- } catch (Exception ex) {
- LOGGER.error("Unsupported type: " + streamsDatum.getDocument().getClass(), ex);
- }
- }
- return dbObject;
- }
-
- private synchronized void connectToMongo() {
-
- ServerAddress serverAddress = new ServerAddress(config.getHost(), config.getPort().intValue());
-
- if (StringUtils.isNotEmpty(config.getUser()) && StringUtils.isNotEmpty(config.getPassword())) {
- MongoCredential credential =
- MongoCredential.createCredential(config.getUser(), config.getDb(), config.getPassword().toCharArray());
- client = new MongoClient(serverAddress, Stream.of(credential).collect(Collectors.toList()));
- } else {
- client = new MongoClient(serverAddress);
- }
-
- db = client.getDB(config.getDb());
-
- if (!db.collectionExists(config.getCollection())) {
- db.createCollection(config.getCollection(), null);
- }
-
- collection = db.getCollection(config.getCollection());
- }
-
-
-}
diff --git a/streams-contrib/streams-persist-mongo/src/main/jsonschema/org/apache/streams/mongo/MongoConfiguration.json b/streams-contrib/streams-persist-mongo/src/main/jsonschema/org/apache/streams/mongo/MongoConfiguration.json
deleted file mode 100644
index 88857c6..0000000
--- a/streams-contrib/streams-persist-mongo/src/main/jsonschema/org/apache/streams/mongo/MongoConfiguration.json
+++ /dev/null
@@ -1,36 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.mongo.MongoConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "host": {
- "type": "string",
- "description": "Hbase host"
- },
- "port": {
- "type": "integer",
- "description": "ZK Port"
- },
- "user": {
- "type": "string",
- "description": "User"
- },
- "password": {
- "type": "string",
- "description": "Password"
- },
- "db": {
- "type": "string",
- "description": "DB"
- },
- "collection": {
- "type": "string",
- "description": "Collection"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-mongo/src/main/resources/components.dot b/streams-contrib/streams-persist-mongo/src/main/resources/components.dot
deleted file mode 100644
index 97e095a..0000000
--- a/streams-contrib/streams-persist-mongo/src/main/resources/components.dot
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- processors [label="processors", shape="circle"];
-
- subgraph cluster_persisters {
- label="persisters";
- persisters_mongo_reader [label="MongoPersistReader"]
- persisters_mongo_writer [label="MongoPersistWriter"]
- }
-
- subgraph cluster_dbs {
- label="dbs";
- mongo [label="mongo", shape="cylinder"]
- }
-
- generators -> providers
- providers -> processors
- processors -> persisters_mongo_writer [label="StreamsDatum"]
- persisters_mongo_reader -> processors [label="StreamsDatum[String]"]
- mongo -> persisters_mongo_reader
- persisters_mongo_writer -> mongo
-
-}
diff --git a/streams-contrib/streams-persist-mongo/src/main/resources/mongo.conf b/streams-contrib/streams-persist-mongo/src/main/resources/mongo.conf
deleted file mode 100644
index 531f771..0000000
--- a/streams-contrib/streams-persist-mongo/src/main/resources/mongo.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"mongo": {
- "host": "localhost",
- "port": 27017,
- "db": "local",
- "collection": "startup_log"
-}
diff --git a/streams-contrib/streams-persist-mongo/src/site/markdown/index.md b/streams-contrib/streams-persist-mongo/src/site/markdown/index.md
deleted file mode 100644
index 4fdc9b8..0000000
--- a/streams-contrib/streams-persist-mongo/src/site/markdown/index.md
+++ /dev/null
@@ -1,23 +0,0 @@
-streams-persist-mongo
-=====================
-
-Read/write to/from MongoDB
-
-## Configuration
-
-| Schema |
-|--------|
-| [MongoConfiguration.json](org/apache/streams/mongo/MongoConfiguration.json "MongoConfiguration.json") [MongoConfiguration.html](apidocs/org/apache/streams/mongo/MongoConfiguration.html "javadoc") |
-
-## Components
-
-
-
-| Class |
-|-------|
-| MongoPersistReader [MongoPersistReader.html](apidocs/org/apache/streams/mongo/MongoPersistReader.html "javadoc")
-| MongoPersistWriter [MongoPersistWriter.html](apidocs/org/apache/streams/mongo/MongoPersistWriter.html "javadoc")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-mongo/src/site/markdown/mongo.md b/streams-contrib/streams-persist-mongo/src/site/markdown/mongo.md
deleted file mode 100644
index 8201bf9..0000000
--- a/streams-contrib/streams-persist-mongo/src/site/markdown/mongo.md
+++ /dev/null
@@ -1,34 +0,0 @@
-## Mongo
-
-Start mongo via docker with the docker maven plugin:
-
- mvn -PdockerITs docker:start
-
-Confirm that elasticsearch is running:
-
- docker ps
-
-Confirm that host and post(s) are in property file:
-
- cat mongo.properties
-
-Create a local file `elasticsearch.conf` with cluster details:
-
- mongo {
- host = ${mongo.tcp.host}
- port = ${mongo.tcp.port}
- }
-
-When configuring a stream, include these files:
-
- include "mongo.properties"
- include "mongo.conf"
-
-Supply application-specific configuration as well:
-
- mongo {
- db: "",
- collection: ""
- }
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-persist-mongo/src/site/site.xml b/streams-contrib/streams-persist-mongo/src/site/site.xml
deleted file mode 100644
index ee8d506..0000000
--- a/streams-contrib/streams-persist-mongo/src/site/site.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project>
- <body>
- <links name="Help">
- <item name="Mongo" href="mongo.html"/>
- </links>
- </body>
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-mongo/src/test/java/org/apache/streams/mongo/test/MongoPersistIT.java b/streams-contrib/streams-persist-mongo/src/test/java/org/apache/streams/mongo/test/MongoPersistIT.java
deleted file mode 100644
index bf7d80e..0000000
--- a/streams-contrib/streams-persist-mongo/src/test/java/org/apache/streams/mongo/test/MongoPersistIT.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.mongo.test;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.mongo.MongoConfiguration;
-import org.apache.streams.mongo.MongoPersistReader;
-import org.apache.streams.mongo.MongoPersistWriter;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test writing documents
- */
-public class MongoPersistIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MongoPersistIT.class);
-
- private ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private MongoConfiguration testConfiguration;
-
- private int count = 0;
-
- @BeforeClass
- public void setup() throws Exception {
-
- Config reference = ConfigFactory.load();
- File conf_file = new File("target/test-classes/MongoPersistIT.conf");
- assert(conf_file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(MongoConfiguration.class).detectConfiguration(typesafe, "mongo");
-
- }
-
- @Test
- public void testMongoPersist() throws Exception {
-
- MongoPersistWriter writer = new MongoPersistWriter(testConfiguration);
-
- writer.prepare(null);
-
- InputStream testActivityFolderStream = MongoPersistIT.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = MongoPersistIT.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- activity.getAdditionalProperties().remove("$license");
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- writer.write( datum );
- LOGGER.info("Wrote: " + activity.getVerb() );
- count++;
- }
-
- LOGGER.info("Total Written: {}", count );
-
- assertEquals( 89, count );
-
- writer.cleanUp();
-
- MongoPersistReader reader = new MongoPersistReader(testConfiguration);
-
- reader.prepare(null);
-
- StreamsResultSet resultSet = reader.readAll();
-
- LOGGER.info("Total Read: {}", resultSet.size() );
-
- assertEquals( 89, resultSet.size() );
-
- }
-}
diff --git a/streams-contrib/streams-persist-mongo/src/test/resources/MongoPersistIT.conf b/streams-contrib/streams-persist-mongo/src/test/resources/MongoPersistIT.conf
deleted file mode 100644
index f356bb0..0000000
--- a/streams-contrib/streams-persist-mongo/src/test/resources/MongoPersistIT.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-mongo {
- host = ${mongo.tcp.host}
- port = ${mongo.tcp.port}
- db = "mongo_persist_it"
- collection = "activity"
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-neo4j/pom.xml b/streams-contrib/streams-persist-neo4j/pom.xml
deleted file mode 100644
index b923d23..0000000
--- a/streams-contrib/streams-persist-neo4j/pom.xml
+++ /dev/null
@@ -1,260 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-neo4j</artifactId>
- <name>streams-persist-neo4j</name>
-
- <properties>
- <neo4j-driver.version>1.4.3</neo4j-driver.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- <version>1.2.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-converters</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-http</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-persist-graph</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.antlr</groupId>
- <artifactId>stringtemplate</artifactId>
- <version>4.0.2</version>
- </dependency>
- <dependency>
- <groupId>org.javatuples</groupId>
- <artifactId>javatuples</artifactId>
- <version>1.2</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.neo4j.driver</groupId>
- <artifactId>neo4j-java-driver</artifactId>
- <version>${neo4j-driver.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <version>3.0.1</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.graph.pojo</targetPackage>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-http</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeTypes>test-jar</includeTypes>
- </configuration>
- <executions>
- <execution>
- <id>test-resource-dependencies</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <id>dockerITs</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>skipITs</name>
- <value>false</value>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>io.fabric8</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- <configuration combine.self="override">
- <watchInterval>500</watchInterval>
- <logDate>default</logDate>
- <verbose>true</verbose>
- <autoPull>on</autoPull>
- <images>
- <image>
- <name>neo4j:3.0.6</name>
- <alias>neo4j</alias>
- <run>
- <env>
- <NEO4J_AUTH>none</NEO4J_AUTH>
- </env>
- <namingStrategy>none</namingStrategy>
- <ports>
- <port>${neo4j.http.host}:${neo4j.http.port}:7474</port>
- <port>${neo4j.tcp.host}:${neo4j.tcp.port}:7687</port>
- </ports>
- <portPropertyFile>neo4j.properties</portPropertyFile>
- <wait>
- <log>neo4j startup</log>
- <http>
- <url>http://${neo4j.http.host}:${neo4j.http.port}</url>
- <method>GET</method>
- <status>200</status>
- </http>
- <time>20000</time>
- <kill>1000</kill>
- <shutdown>500</shutdown>
- <!--<tcp>-->
- <!--<host>${es.transport.host}</host>-->
- <!--<ports>-->
- <!--<port>${es.transport.port}</port>-->
- <!--</ports>-->
- <!--</tcp>-->
- </wait>
- <log>
- <enabled>true</enabled>
- <date>default</date>
- <color>cyan</color>
- </log>
- </run>
- <watch>
- <mode>none</mode>
- </watch>
- </image>
-
- </images>
- </configuration>
-
- </plugin>
-
- </plugins>
- </build>
-
- </profile>
- </profiles>
-
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/CypherQueryGraphHelper.java b/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/CypherQueryGraphHelper.java
deleted file mode 100644
index 7293d14..0000000
--- a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/CypherQueryGraphHelper.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.neo4j;
-
-import org.apache.streams.graph.QueryGraphHelper;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.util.PropertyUtil;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import org.apache.commons.lang3.StringEscapeUtils;
-import org.javatuples.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.stringtemplate.v4.ST;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * Supporting class for interacting with neo4j via rest API
- */
-public class CypherQueryGraphHelper implements QueryGraphHelper, Serializable {
-
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private static final Logger LOGGER = LoggerFactory.getLogger(CypherQueryGraphHelper.class);
-
- public static final String getVertexLongIdStatementTemplate = "MATCH (v) WHERE ID(v) = <id> RETURN v";
- public static final String getVertexStringIdStatementTemplate = "MATCH (v {id: '<id>'} ) RETURN v";
- public static final String getVerticesLabelIdStatementTemplate = "MATCH (v:<type>) RETURN v";
-
- public final static String createVertexStatementTemplate = "MATCH (x {id: '<id>'}) "+
- "CREATE UNIQUE (v:`<type>` { props }) "+
- "ON CREATE SET v <labels> "+
- "RETURN v";
-
- public final static String mergeVertexStatementTemplate = "MERGE (v:`<type>` {id: '<id>'}) "+
- "ON CREATE SET v <labels>, v = { props }, v.`@timestamp` = timestamp() "+
- "ON MATCH SET v <labels>, v = { props }, v.`@timestamp` = timestamp() "+
- "RETURN v";
-
- public final static String createEdgeStatementTemplate = "MATCH (s:`<s_type>` {id: '<s_id>'}),(d:`<d_type>` {id: '<d_id>'}) "+
- "CREATE UNIQUE (s)-[r:`<r_type>` <r_props>]->(d) "+
- "RETURN r";
-
- public Pair<String, Map<String, Object>> getVertexRequest(String streamsId) {
-
- ST getVertex = new ST(getVertexStringIdStatementTemplate);
- getVertex.add("id", streamsId);
-
- Pair<String, Map<String, Object>> queryPlusParameters = new Pair(getVertex.render(), null);
-
- LOGGER.debug("getVertexRequest", queryPlusParameters.toString());
-
- return queryPlusParameters;
- }
-
- /**
- * getVertexRequest.
- * @param vertexId numericId
- * @return pair (streamsId, parameterMap)
- */
- public Pair<String, Map<String, Object>> getVertexRequest(Long vertexId) {
-
- ST getVertex = new ST(getVertexLongIdStatementTemplate);
- getVertex.add("id", vertexId);
-
- Pair<String, Map<String, Object>> queryPlusParameters = new Pair(getVertex.render(), null);
-
- LOGGER.debug("getVertexRequest", queryPlusParameters.toString());
-
- return queryPlusParameters;
-
- }
-
- /**
- * createVertexRequest.
- * @param activityObject activityObject
- * @return pair (query, parameterMap)
- */
- public Pair<String, Map<String, Object>> createVertexRequest(ActivityObject activityObject) {
-
- Objects.requireNonNull(activityObject.getObjectType());
-
- List<String> labels = getLabels(activityObject);
-
- ST createVertex = new ST(createVertexStatementTemplate);
- createVertex.add("id", activityObject.getId());
- createVertex.add("type", activityObject.getObjectType());
-
- if ( labels.size() > 0 ) {
- createVertex.add("labels", String.join(" ", labels));
- }
-
- String query = createVertex.render();
-
- ObjectNode object = MAPPER.convertValue(activityObject, ObjectNode.class);
- Map<String, Object> props = PropertyUtil.getInstance(MAPPER).flattenToMap(object);
-
- Pair<String, Map<String, Object>> queryPlusParameters = new Pair(createVertex.render(), props);
-
- LOGGER.debug("createVertexRequest: ({},{})", query, props);
-
- return queryPlusParameters;
- }
-
- /**
- * getVerticesRequest gets all vertices with a label.
- * @param labelId labelId
- * @return pair (query, parameterMap)
- */
- public Pair<String, Map<String, Object>> getVerticesRequest(String labelId) {
- ST getVertex = new ST(getVerticesLabelIdStatementTemplate);
- getVertex.add("type", labelId);
-
- Pair<String, Map<String, Object>> queryPlusParameters = new Pair(getVertex.render(), null);
-
- LOGGER.debug("getVertexRequest", queryPlusParameters.toString());
-
- return queryPlusParameters;
- }
-
- /**
- * mergeVertexRequest.
- * @param activityObject activityObject
- * @return pair (query, parameterMap)
- */
- public Pair<String, Map<String, Object>> mergeVertexRequest(ActivityObject activityObject) {
-
- Objects.requireNonNull(activityObject.getObjectType());
-
- Pair queryPlusParameters = new Pair(null, new HashMap<>());
-
- List<String> labels = getLabels(activityObject);
-
- ST mergeVertex = new ST(mergeVertexStatementTemplate);
- mergeVertex.add("id", activityObject.getId());
- mergeVertex.add("type", activityObject.getObjectType());
- if ( labels.size() > 0 ) {
- mergeVertex.add("labels", String.join(" ", labels));
- }
- String query = mergeVertex.render();
-
- ObjectNode object = MAPPER.convertValue(activityObject, ObjectNode.class);
- Map<String, Object> props = PropertyUtil.getInstance(MAPPER).flattenToMap(object);
-
- LOGGER.debug("mergeVertexRequest: ({},{})", query, props);
-
- queryPlusParameters = queryPlusParameters.setAt0(query);
- queryPlusParameters = queryPlusParameters.setAt1(props);
-
- return queryPlusParameters;
- }
-
- /**
- * createActorObjectEdge.
- * @param activity activity
- * @return pair (query, parameterMap)
- */
- public Pair<String, Map<String, Object>> createActorObjectEdge(Activity activity) {
-
- Pair queryPlusParameters = new Pair(null, new HashMap<>());
-
- ObjectNode object = MAPPER.convertValue(activity, ObjectNode.class);
- Map<String, Object> props = PropertyUtil.getInstance(MAPPER).flattenToMap(object);
-
- ST mergeEdge = new ST(createEdgeStatementTemplate);
- mergeEdge.add("s_id", activity.getActor().getId());
- mergeEdge.add("s_type", activity.getActor().getObjectType());
- mergeEdge.add("d_id", activity.getObject().getId());
- mergeEdge.add("d_type", activity.getObject().getObjectType());
- mergeEdge.add("r_id", activity.getId());
- mergeEdge.add("r_type", activity.getVerb());
- mergeEdge.add("r_props", getActorObjectEdgePropertyCreater(props));
-
- String statement = mergeEdge.render();
- queryPlusParameters = queryPlusParameters.setAt0(statement);
- queryPlusParameters = queryPlusParameters.setAt1(props);
-
- LOGGER.debug("createActorObjectEdge: ({},{})", statement, props);
-
- return queryPlusParameters;
- }
-
- /**
- * createActorTargetEdge.
- * @param activity activity
- * @return pair (query, parameterMap)
- */
- public Pair<String, Map<String, Object>> createActorTargetEdge(Activity activity) {
-
- Pair queryPlusParameters = new Pair(null, new HashMap<>());
-
- ObjectNode object = MAPPER.convertValue(activity, ObjectNode.class);
- Map<String, Object> props = PropertyUtil.getInstance(MAPPER).flattenToMap(object);
-
- ST mergeEdge = new ST(createEdgeStatementTemplate);
- mergeEdge.add("s_id", activity.getActor().getId());
- mergeEdge.add("s_type", activity.getActor().getObjectType());
- mergeEdge.add("d_id", activity.getTarget().getId());
- mergeEdge.add("d_type", activity.getTarget().getObjectType());
- mergeEdge.add("r_id", activity.getId());
- mergeEdge.add("r_type", activity.getVerb());
- mergeEdge.add("r_props", getActorTargetEdgePropertyCreater(props));
-
- String statement = mergeEdge.render();
- queryPlusParameters = queryPlusParameters.setAt0(statement);
- queryPlusParameters = queryPlusParameters.setAt1(props);
-
- LOGGER.debug("createActorObjectEdge: ({},{})", statement, props);
-
- return queryPlusParameters;
- }
-
- /**
- * getPropertyValueSetter.
- * @param map paramMap
- * @return PropertyValueSetter string
- */
- public static String getPropertyValueSetter(Map<String, Object> map, String symbol) {
- StringBuilder builder = new StringBuilder();
- for( Map.Entry<String, Object> entry : map.entrySet()) {
- if( entry.getValue() instanceof String ) {
- String propVal = (String)(entry.getValue());
- builder.append("," + symbol + ".`" + entry.getKey() + "` = '" + StringEscapeUtils.escapeJava(propVal) + "'");
- }
- }
- return builder.toString();
- }
-
- /**
- * getPropertyParamSetter.
- * @param map paramMap
- * @return PropertyParamSetter string
- */
- public static String getPropertyParamSetter(Map<String, Object> map, String symbol) {
- StringBuilder builder = new StringBuilder();
- for( Map.Entry<String, Object> entry : map.entrySet()) {
- if( entry.getValue() instanceof String ) {
- String propVal = (String)(entry.getValue());
- builder.append("," + symbol + ".`" + entry.getKey() + "` = '" + StringEscapeUtils.escapeJava(propVal) + "'");
- }
- }
- return builder.toString();
- }
-
- /**
- * getPropertyCreater.
- * @param map paramMap
- * @return PropertyCreater string
- */
- public static String getPropertyCreater(Map<String, Object> map) {
- StringBuilder builder = new StringBuilder();
- builder.append("{ ");
- List<String> parts = new ArrayList<>();
- for( Map.Entry<String, Object> entry : map.entrySet()) {
- if( entry.getValue() instanceof String ) {
- String propVal = (String) (entry.getValue());
- parts.add("`"+entry.getKey() + "`:'" + StringEscapeUtils.escapeJava(propVal) + "'");
- }
- }
- builder.append(String.join(" ", parts));
- builder.append(" }");
- return builder.toString();
- }
-
- private String getActorObjectEdgePropertyCreater(Map<String, Object> map) {
- StringBuilder builder = new StringBuilder();
- builder.append("{ ");
- List<String> parts = new ArrayList<>();
- for( Map.Entry<String, Object> entry : map.entrySet()) {
- if( entry.getValue() instanceof String ) {
- String propVal = (String) (entry.getValue());
- if( !entry.getKey().contains(".")) {
- parts.add("`"+entry.getKey() + "`: '" + StringEscapeUtils.escapeJava(propVal) + "'");
- }
- }
- }
- builder.append(String.join(", ", parts));
- builder.append(" }");
- return builder.toString();
- }
-
- private String getActorTargetEdgePropertyCreater(Map<String, Object> map) {
- StringBuilder builder = new StringBuilder();
- builder.append("{ ");
- List<String> parts = new ArrayList<>();
- for( Map.Entry<String, Object> entry : map.entrySet()) {
- if( entry.getValue() instanceof String ) {
- String propVal = (String) (entry.getValue());
- if( !entry.getKey().contains(".")) {
- parts.add("`"+entry.getKey() + "`: '" + StringEscapeUtils.escapeJava(propVal) + "'");
- } else if( entry.getKey().startsWith("object.") && !entry.getKey().contains(".id")) {
- parts.add("`"+entry.getKey().substring("object.".length()) + "`: '" + StringEscapeUtils.escapeJava(propVal) + "'");
- }
- }
- }
- builder.append(String.join(", ", parts));
- builder.append(" }");
- return builder.toString();
- }
-
- /**
- * getLabels.
- * @param activityObject activityObject
- * @return PropertyCreater string
- */
- public static List<String> getLabels(ActivityObject activityObject) {
- List<String> labels = Collections.singletonList(":streams");
- if ( activityObject.getAdditionalProperties().containsKey("labels") ) {
- List<String> extraLabels = (List<String>)activityObject.getAdditionalProperties().get("labels");
- for ( String extraLabel : extraLabels ) {
- labels.add(":" + extraLabel);
- }
- }
- return labels;
- }
-
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/Neo4jPersistUtil.java b/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/Neo4jPersistUtil.java
deleted file mode 100644
index 4fe2c1c..0000000
--- a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/Neo4jPersistUtil.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.neo4j;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.neo4j.bolt.Neo4jBoltPersistWriter;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-
-import org.apache.commons.lang3.StringUtils;
-import org.javatuples.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Created by steve on 1/2/17.
- */
-public class Neo4jPersistUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(Neo4jBoltPersistWriter.class);
-
- private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private static CypherQueryGraphHelper helper = new CypherQueryGraphHelper();
-
- public static List<Pair<String, Map<String, Object>>> prepareStatements(StreamsDatum entry) throws Exception {
-
- List<Pair<String, Map<String, Object>>> statements = new ArrayList<>();
-
- String id = entry.getId();
- Activity activity = null;
- ActivityObject activityObject = null;
- Object document = entry.getDocument();
-
- if (document instanceof Activity) {
- activity = (Activity) document;
- } else if (document instanceof ActivityObject) {
- activityObject = (ActivityObject) document;
- } else {
- ObjectNode objectNode;
- if (document instanceof ObjectNode) {
- objectNode = (ObjectNode) document;
- } else if ( document instanceof String) {
- try {
- objectNode = mapper.readValue((String) document, ObjectNode.class);
- } catch (IOException ex) {
- LOGGER.error("Can't handle input: ", entry);
- throw ex;
- }
- } else {
- LOGGER.error("Can't handle input: ", entry);
- throw new Exception("Can't create statements from datum.");
- }
-
- if ( objectNode.get("verb") != null ) {
- try {
- activity = mapper.convertValue(objectNode, Activity.class);
- activityObject = activity.getObject();
- } catch (Exception ex) {
- activityObject = mapper.convertValue(objectNode, ActivityObject.class);
- }
- } else {
- activityObject = mapper.convertValue(objectNode, ActivityObject.class);
- }
-
- }
-
- Preconditions.checkArgument(activity != null ^ activityObject != null);
-
- if ( activityObject != null && !Strings.isNullOrEmpty(activityObject.getId())) {
-
- statements.add(vertexStatement(activityObject));
-
- } else if ( activity != null && !Strings.isNullOrEmpty(activity.getId())) {
-
- statements.addAll(vertexStatements(activity));
-
- statements.addAll(edgeStatements(activity));
-
- }
-
- return statements;
- }
-
- public static List<Pair<String, Map<String, Object>>> vertexStatements(Activity activity) {
- List<Pair<String, Map<String, Object>>> statements = new ArrayList<>();;
- ActivityObject actor = activity.getActor();
- ActivityObject object = activity.getObject();
- ActivityObject target = activity.getTarget();
-
- if (actor != null && StringUtils.isNotBlank(actor.getId())) {
- Pair<String, Map<String, Object>> actorStatement = vertexStatement(actor);
- statements.add(actorStatement);
- }
-
- if (object != null && StringUtils.isNotBlank(object.getId())) {
- Pair<String, Map<String, Object>> objectStatement = vertexStatement(object);
- statements.add(objectStatement);
- }
-
- if (target != null && StringUtils.isNotBlank(target.getId())) {
- Pair<String, Map<String, Object>> targetStatement = vertexStatement(target);
- statements.add(targetStatement);
- }
-
- return statements;
- }
-
- public static List<Pair<String, Map<String, Object>>> edgeStatements(Activity activity) {
- List<Pair<String, Map<String, Object>>> statements = new ArrayList<>();;
- ActivityObject actor = activity.getActor();
- ActivityObject object = activity.getObject();
- ActivityObject target = activity.getTarget();
-
- if (StringUtils.isNotBlank(actor.getId()) && object != null && StringUtils.isNotBlank(object.getId())) {
- Pair<String, Map<String, Object>> actorObjectEdgeStatement = helper.createActorObjectEdge(activity);
- Map<String, Object> props = new HashMap<>();
- props.put("props", actorObjectEdgeStatement.getValue1());
- actorObjectEdgeStatement = actorObjectEdgeStatement.setAt1(props);
- statements.add(actorObjectEdgeStatement);
- }
-
- if (StringUtils.isNotBlank(actor.getId()) && target != null && StringUtils.isNotBlank(target.getId())) {
- Pair<String, Map<String, Object>> actorTargetEdgeStatement = helper.createActorTargetEdge(activity);
- Map<String, Object> props = new HashMap<>();
- props.put("props", actorTargetEdgeStatement.getValue1());
- actorTargetEdgeStatement = actorTargetEdgeStatement.setAt1(props);
- statements.add(actorTargetEdgeStatement);
- }
-
- return statements;
- }
-
- public static Pair<String, Map<String, Object>> vertexStatement(ActivityObject activityObject) {
- Pair<String, Map<String, Object>> mergeVertexRequest = helper.mergeVertexRequest(activityObject);
- Map<String, Object> props = new HashMap<>();
- props.put("props", mergeVertexRequest.getValue1());
- mergeVertexRequest = mergeVertexRequest.setAt1(props);
- return mergeVertexRequest;
- }
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/bolt/Neo4jBoltClient.java b/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/bolt/Neo4jBoltClient.java
deleted file mode 100644
index 2bbbc7c..0000000
--- a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/bolt/Neo4jBoltClient.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.neo4j.bolt;
-
-import org.apache.streams.neo4j.Neo4jConfiguration;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.commons.lang3.StringUtils;
-import org.neo4j.driver.v1.AuthToken;
-import org.neo4j.driver.v1.AuthTokens;
-import org.neo4j.driver.v1.Driver;
-import org.neo4j.driver.v1.GraphDatabase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-
-public class Neo4jBoltClient {
-
- private static final Logger LOGGER = LoggerFactory
- .getLogger(Neo4jBoltClient.class);
-
- private Driver client;
-
- public Neo4jConfiguration config;
-
- private Neo4jBoltClient(Neo4jConfiguration neo4jConfiguration) {
- this.config = neo4jConfiguration;
- try {
- this.start();
- } catch (Exception e) {
- e.printStackTrace();
- this.client = null;
- }
- }
-
- private static Map<Neo4jConfiguration, Neo4jBoltClient> INSTANCE_MAP = new ConcurrentHashMap<Neo4jConfiguration, Neo4jBoltClient>();
-
- public static Neo4jBoltClient getInstance(Neo4jConfiguration neo4jConfiguration) {
- if ( INSTANCE_MAP != null &&
- INSTANCE_MAP.size() > 0 &&
- INSTANCE_MAP.containsKey(neo4jConfiguration)) {
- return INSTANCE_MAP.get(neo4jConfiguration);
- } else {
- Neo4jBoltClient instance = new Neo4jBoltClient(neo4jConfiguration);
- if( instance != null && instance.client != null ) {
- INSTANCE_MAP.put(neo4jConfiguration, instance);
- return instance;
- } else {
- return null;
- }
- }
- }
-
- public void start() throws Exception {
-
- Objects.nonNull(config);
- assertThat("config.getScheme().startsWith(\"tcp\")", config.getScheme().startsWith("tcp"));
-
- LOGGER.info("Neo4jConfiguration.start {}", config);
-
- AuthToken authToken = null;
- if( StringUtils.isNotBlank(config.getUsername()) && StringUtils.isNotBlank(config.getPassword())) {
- authToken = AuthTokens.basic( config.getUsername(), config.getPassword() );
- }
-
- if( authToken == null ) {
- client = GraphDatabase.driver("bolt://" + config.getHosts().get(0) + ":" + config.getPort());
- } else {
- client = GraphDatabase.driver("bolt://" + config.getHosts().get(0) + ":" + config.getPort(), authToken);
- }
-
- Objects.nonNull(client);
-
- }
-
- public void stop() throws Exception {
- this.client.session().close();
- this.client = null;
- }
-
- public Neo4jConfiguration config() {
- return config;
- }
-
- public Driver client() {
- return client;
- }
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/bolt/Neo4jBoltPersistReader.java b/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/bolt/Neo4jBoltPersistReader.java
deleted file mode 100644
index 25b338a..0000000
--- a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/bolt/Neo4jBoltPersistReader.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.neo4j.bolt;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.neo4j.Neo4jReaderConfiguration;
-import org.apache.streams.util.PropertyUtil;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Queues;
-
-import org.joda.time.DateTime;
-import org.neo4j.driver.internal.value.NodeValue;
-import org.neo4j.driver.internal.value.RelationshipValue;
-import org.neo4j.driver.internal.value.StringValue;
-import org.neo4j.driver.v1.Record;
-import org.neo4j.driver.v1.Session;
-import org.neo4j.driver.v1.StatementResult;
-import org.neo4j.driver.v1.Transaction;
-import org.neo4j.driver.v1.Value;
-import org.neo4j.driver.v1.types.Node;
-import org.neo4j.driver.v1.types.Relationship;
-import org.neo4j.driver.v1.util.Function;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.math.BigInteger;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Queue;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import javax.annotation.Nullable;
-
-
-/**
- * Neo4jBoltPersistReader reads documents from neo4j.
- */
-public class Neo4jBoltPersistReader implements StreamsPersistReader {
-
- public static final String STREAMS_ID = "CassandraPersistReader";
-
- public static final Logger LOGGER = LoggerFactory.getLogger(Neo4jBoltPersistReader.class);
-
- protected volatile Queue<StreamsDatum> persistQueue;
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private ExecutorService executor;
- private CompletableFuture<Boolean> readerTaskFuture = new CompletableFuture<>();
-
- private Neo4jReaderConfiguration config;
-
- protected Neo4jBoltClient client;
-
-// protected Cluster cluster;
-// protected Session session;
-//
-// protected String keyspace;
-// protected String table;
- protected StatementResult statementResult;
-
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- /**
- * Neo4jBoltPersistReader constructor - resolves Neo4jReaderConfiguration from JVM 'neo4j'.
- */
- public Neo4jBoltPersistReader() {
- this.config = new ComponentConfigurator<>(Neo4jReaderConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("neo4j"));
- }
-
- /**
- * Neo4jBoltPersistReader constructor - uses supplied Neo4jReaderConfiguration.
- * @param config config
- */
- public Neo4jBoltPersistReader(Neo4jReaderConfiguration config) {
- this.config = config;
- }
-
- /**
- * Neo4jBoltPersistReader constructor - uses supplied persistQueue.
- * @param persistQueue persistQueue
- */
- public Neo4jBoltPersistReader(Queue<StreamsDatum> persistQueue) {
- this();
- this.persistQueue = persistQueue;
- }
-
- public void setPersistQueue(Queue<StreamsDatum> persistQueue) {
- this.persistQueue = persistQueue;
- }
-
- public Queue<StreamsDatum> getPersistQueue() {
- return persistQueue;
- }
-
- public void stop() {
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- if( configurationObject instanceof Neo4jReaderConfiguration ) {
- this.config = (Neo4jReaderConfiguration) configurationObject;
- }
- this.client = Neo4jBoltClient.getInstance(this.config);
-
- persistQueue = constructQueue();
-
- executor = Executors.newSingleThreadExecutor();
- }
-
- @Override
- public void cleanUp() {
- stop();
- }
-
- protected Optional<StreamsDatum> buildDatum(Record record) {
- ObjectNode objectNode;
-
- if( record != null ) {
- ObjectNode valueJson = null;
- Map<String, ObjectNode> valueJsons = record.asMap(neo4jObjectNodeFunction);
- if( valueJsons.size() == 1) {
- valueJson = valueJsons.get(valueJsons.keySet().iterator().next());
- }
- objectNode = PropertyUtil.getInstance(mapper).unflattenObjectNode(valueJson);
- return Optional.of(new StreamsDatum(objectNode));
- }
-
- return Optional.empty();
- }
-
- @Override
- public StreamsResultSet readAll() {
-
- Session session = null;
-
- String query = config.getQuery();
- Map<String, Object> params = mapper.convertValue(config.getParams(), Map.class);
-
- try {
- session = client.client().session();
- Transaction transaction = session.beginTransaction();
-
- this.statementResult = client.client().session().beginTransaction().run(query, params);
-
- while( statementResult.hasNext()) {
- Record record = statementResult.next();
- Optional<StreamsDatum> datum = buildDatum(record);
- if( datum.isPresent()) {
- write(datum.get());
- }
- }
-
- } catch(Exception ex) {
- LOGGER.warn("Exception", ex);
- } finally {
- if( session != null ) {
- session.close();
- }
- }
- return readCurrent();
-
- }
-
- @Override
- public void startStream() {
- LOGGER.debug("startStream");
- Neo4jBoltPersistReaderTask readerTask = new Neo4jBoltPersistReaderTask(this);
-
- CompletableFuture.runAsync(readerTask, executor);
-
- try {
- if (readerTaskFuture.get()) {
- executor.shutdown();
- }
- } catch (InterruptedException ex) {
- LOGGER.trace("Interrupt", ex);
- } catch (ExecutionException ex) {
- LOGGER.trace("Execution exception", ex);
- }
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- try {
- lock.writeLock().lock();
- current = new StreamsResultSet(persistQueue);
- current.setCounter(new DatumStatusCounter());
- persistQueue = constructQueue();
- } finally {
- lock.writeLock().unlock();
- }
-
- return current;
- }
-
- protected void write(StreamsDatum entry) {
- boolean success;
- do {
- try {
- lock.readLock().lock();
- success = persistQueue.offer(entry);
- Thread.yield();
- } finally {
- lock.readLock().unlock();
- }
- }
- while (!success);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return !executor.isTerminated() || !executor.isShutdown();
- }
-
- private Queue<StreamsDatum> constructQueue() {
- return Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
- }
-
- private static String readAllStatement() {
- return "MATCH (v:streams)";
- }
-
- public class Neo4jBoltPersistReaderTask implements Runnable {
-
- private Neo4jBoltPersistReader reader;
-
- public Neo4jBoltPersistReaderTask(Neo4jBoltPersistReader reader) {
- this.reader = reader;
- }
-
- @Override
- public void run() {
- try {
- while (reader.statementResult.hasNext()) {
- Record record = statementResult.next();
- Optional<StreamsDatum> datum = reader.buildDatum(record);
- if( datum.isPresent() ) {
- reader.write(datum.get());
- }
- }
- } finally {
- readerTaskFuture.complete(true);
- }
- }
- }
-
- Function<Value, ObjectNode> neo4jObjectNodeFunction = new Function<Value, ObjectNode>() {
-
- @Nullable
- @Override
- public ObjectNode apply(@Nullable Value value) {
- ObjectNode resultNode = null;
- if (value instanceof StringValue) {
- StringValue stringValue = (StringValue) value;
- String string = stringValue.asLiteralString();
- try {
- resultNode = mapper.readValue(string, ObjectNode.class);
- } catch (IOException ex) {
- LOGGER.error("IOException", ex);
- }
- } else if ( value instanceof NodeValue) {
- NodeValue nodeValue = (NodeValue) value;
- Node node = nodeValue.asNode();
- Map<String, Object> nodeMap = node.asMap();
- resultNode = PropertyUtil.getInstance(mapper).unflattenMap(nodeMap);
- } else if (value instanceof RelationshipValue) {
- RelationshipValue relationshipValue = (RelationshipValue) value;
- Relationship relationship = relationshipValue.asRelationship();
- Map<String, Object> relationshipMap = relationship.asMap();
- resultNode = PropertyUtil.getInstance(mapper).unflattenMap(relationshipMap);
- }
- return resultNode;
- }
- };
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/bolt/Neo4jBoltPersistWriter.java b/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/bolt/Neo4jBoltPersistWriter.java
deleted file mode 100644
index a35a60e..0000000
--- a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/bolt/Neo4jBoltPersistWriter.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.neo4j.bolt;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.neo4j.Neo4jConfiguration;
-import org.apache.streams.neo4j.Neo4jPersistUtil;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import org.javatuples.Pair;
-import org.neo4j.driver.v1.Session;
-import org.neo4j.driver.v1.StatementResult;
-import org.neo4j.driver.v1.Transaction;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * Created by sblackmon on 12/16/16.
- */
-public class Neo4jBoltPersistWriter implements StreamsPersistWriter {
-
- private Neo4jConfiguration config;
-
- Neo4jBoltClient client;
-
- private static final Logger LOGGER = LoggerFactory.getLogger(Neo4jBoltPersistWriter.class);
-
- private static ObjectMapper mapper;
-
- public Neo4jBoltPersistWriter(Neo4jConfiguration config) {
- this.config = config;
-
- }
-
- @Override
- public String getId() {
- return Neo4jBoltPersistWriter.class.getSimpleName();
- }
-
- @Override
- public void prepare(Object configurationObject) {
- client = Neo4jBoltClient.getInstance(config);
- }
-
- @Override
- public void cleanUp() {
- //
- }
-
- @Override
- public void write(StreamsDatum entry) {
-
- List<Pair<String, Map<String, Object>>> statements;
- Session session = null;
- try {
- statements = Neo4jPersistUtil.prepareStatements(entry);
- session = client.client().session();
- Transaction transaction = session.beginTransaction();
- for( Pair<String, Map<String, Object>> statement : statements ) {
- StatementResult statementResult = transaction.run( statement.getValue0(), statement.getValue1() );
- LOGGER.debug("StatementResult", statementResult.single());
- }
- transaction.success();
- } catch( Exception ex ) {
- LOGGER.error("Exception", ex);
- } finally {
- if( session != null ) {
- session.close();
- }
- }
- }
-
-
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/http/Neo4jHttpClient.java b/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/http/Neo4jHttpClient.java
deleted file mode 100644
index 85d530e..0000000
--- a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/http/Neo4jHttpClient.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.neo4j.http;
-
-import org.apache.streams.neo4j.Neo4jConfiguration;
-
-import org.apache.http.client.HttpClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-
-public class Neo4jHttpClient {
-
- private static final Logger LOGGER = LoggerFactory
- .getLogger(Neo4jHttpClient.class);
-
- public Neo4jConfiguration config;
-
- private HttpClient client;
-
- private Neo4jHttpClient(Neo4jConfiguration neo4jConfiguration) {
- this.config = neo4jConfiguration;
- try {
- this.start();
- } catch (Exception e) {
- e.printStackTrace();
- this.client = null;
- }
- }
-
- private static Map<Neo4jConfiguration, Neo4jHttpClient> INSTANCE_MAP = new ConcurrentHashMap<Neo4jConfiguration, Neo4jHttpClient>();
-
- public static Neo4jHttpClient getInstance(Neo4jConfiguration neo4jConfiguration) {
- if ( INSTANCE_MAP != null &&
- INSTANCE_MAP.size() > 0 &&
- INSTANCE_MAP.containsKey(neo4jConfiguration)) {
- return INSTANCE_MAP.get(neo4jConfiguration);
- } else {
- Neo4jHttpClient instance = new Neo4jHttpClient(neo4jConfiguration);
- if( instance != null && instance.client != null ) {
- INSTANCE_MAP.put(neo4jConfiguration, instance);
- return instance;
- } else {
- return null;
- }
- }
- }
-
- public void start() throws Exception {
-
- Objects.nonNull(config);
- assertThat("config.getScheme().startsWith(\"http\")", config.getScheme().startsWith("http"));
-
- LOGGER.info("Neo4jConfiguration.start {}", config);
-
- Objects.nonNull(client);
-
- }
-
- public void stop() throws Exception {
- this.client = null;
- }
-
- public Neo4jConfiguration config() {
- return config;
- }
-
- public HttpClient client() {
- return client;
- }
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/http/Neo4jHttpGraphHelper.java b/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/http/Neo4jHttpGraphHelper.java
deleted file mode 100644
index 4c126b8..0000000
--- a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/http/Neo4jHttpGraphHelper.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.neo4j.http;
-
-import org.apache.streams.graph.HttpGraphHelper;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.javatuples.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * Supporting class for interacting with neo4j via rest API.
- */
-public class Neo4jHttpGraphHelper implements HttpGraphHelper {
-
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private static final Logger LOGGER = LoggerFactory.getLogger(Neo4jHttpGraphHelper.class);
-
- private static final String statementKey = "statement";
- private static final String queryKey = "query";
- private static final String paramsKey = "parameters";
- private static final String propsKey = "props";
-
- /**
- * readDataStatement neo4j rest json read data payload.
- *
- * @param queryPlusParameters (query, parameter map)
- * @return ObjectNode
- */
- public ObjectNode readData(Pair<String, Map<String, Object>> queryPlusParameters) {
-
- LOGGER.debug("readData: ", queryPlusParameters);
-
- Objects.requireNonNull(queryPlusParameters);
- Objects.requireNonNull(queryPlusParameters.getValue0());
-
- ObjectNode request = MAPPER.createObjectNode();
-
- request.put(queryKey, queryPlusParameters.getValue0());
-
- if( queryPlusParameters.getValue1() != null && queryPlusParameters.getValue1().size() > 0) {
- ObjectNode params = MAPPER.convertValue(queryPlusParameters.getValue1(), ObjectNode.class);
- request.put(paramsKey, params);
- }
-
- LOGGER.debug("readData: ", request);
-
- return request;
- }
-
- /**
- * writeDataStatement neo4j rest json write data payload.
- *
- * @param queryPlusParameters (query, parameter map)
- * @return ObjectNode
- */
- public ObjectNode writeData(Pair<String, Map<String, Object>> queryPlusParameters) {
-
- LOGGER.debug("writeData: ", queryPlusParameters);
-
- Objects.requireNonNull(queryPlusParameters);
- Objects.requireNonNull(queryPlusParameters.getValue0());
-
- ObjectNode request = MAPPER.createObjectNode();
-
- request.put(statementKey, queryPlusParameters.getValue0());
-
- if( queryPlusParameters.getValue1() != null && queryPlusParameters.getValue1().size() > 0) {
- ObjectNode params = MAPPER.convertValue(queryPlusParameters.getValue1(), ObjectNode.class);
- request.put(paramsKey, params);
- } else {
- request.put(paramsKey, MAPPER.createObjectNode());
- }
-
- LOGGER.debug("writeData: ", request);
-
- return request;
- }
-
-
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/http/Neo4jHttpPersistReader.java b/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/http/Neo4jHttpPersistReader.java
deleted file mode 100644
index 5f7bbf7..0000000
--- a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/http/Neo4jHttpPersistReader.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.neo4j.http;
-
-import org.apache.streams.components.http.HttpConfiguration;
-import org.apache.streams.components.http.HttpProviderConfiguration;
-import org.apache.streams.components.http.provider.SimpleHttpProvider;
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.graph.HttpGraphHelper;
-import org.apache.streams.graph.QueryGraphHelper;
-import org.apache.streams.neo4j.CypherQueryGraphHelper;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.neo4j.CypherQueryResponse;
-import org.apache.streams.neo4j.ItemData;
-import org.apache.streams.neo4j.ItemMetadata;
-import org.apache.streams.neo4j.Neo4jReaderConfiguration;
-import org.apache.streams.util.PropertyUtil;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.javatuples.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * Reads a stream of activityobjects from vertices in a graph database with
- * an http rest endpoint (such as neo4j).
- */
-public class Neo4jHttpPersistReader extends SimpleHttpProvider implements StreamsPersistReader {
-
- public static final String STREAMS_ID = Neo4jHttpPersistReader.class.getCanonicalName();
-
- private static final Logger LOGGER = LoggerFactory.getLogger(Neo4jHttpPersistReader.class);
-
- private Neo4jReaderConfiguration config;
-
- private QueryGraphHelper queryGraphHelper;
- private HttpGraphHelper httpGraphHelper;
-
- private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- /**
- * GraphVertexReader constructor - resolve GraphReaderConfiguration from JVM 'graph'.
- */
- public Neo4jHttpPersistReader() {
- this(new ComponentConfigurator<>(Neo4jReaderConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig().getConfig("neo4j")));
- }
-
- /**
- * GraphVertexReader constructor - use supplied GraphReaderConfiguration.
- * @param configuration GraphReaderConfiguration
- */
- public Neo4jHttpPersistReader(Neo4jReaderConfiguration configuration) {
- super((HttpProviderConfiguration)StreamsJacksonMapper.getInstance().convertValue(configuration, HttpProviderConfiguration.class).withHostname(configuration.getHosts().get(0)));
- super.configuration.setRequestMethod(HttpConfiguration.RequestMethod.POST);
- super.configuration.setResourcePath("/db");
- super.configuration.setResource("data");
- super.configuration.setResourcePostfix("cypher");
- this.config = configuration;
- }
-
- /**
- * prepareHttpRequest
- * @param uri uri
- * @return result
- */
- public HttpRequestBase prepareHttpRequest(URI uri) {
- HttpRequestBase baseRequest = super.prepareHttpRequest(uri);
- HttpPost post = (HttpPost) baseRequest;
- String query = config.getQuery();
- Map<String, Object> params = mapper.convertValue(config.getParams(), Map.class);
- Pair<String, Map<String, Object>> queryPlusParams = new Pair(query, params);
- ObjectNode queryNode = httpGraphHelper.readData(queryPlusParams);
- try {
- String queryJsonString = mapper.writeValueAsString(queryNode);
- HttpEntity entity = new StringEntity(queryJsonString, ContentType.create("application/json"));
- post.setEntity(entity);
- } catch (JsonProcessingException ex) {
- LOGGER.error("JsonProcessingException", ex);
- return null;
- }
- return post;
-
- }
- /**
- * Neo API query returns something like this:
- * { "columns": [ "v" ], "data": [ [ { "data": { props }, etc... } ], [ { "data": { props }, etc... } ] ] }
- *
- * @param jsonNode jsonNode
- * @return result
- */
- public List<ObjectNode> parse(JsonNode jsonNode) {
- List<ObjectNode> results = new ArrayList<>();
-
- ObjectNode root = (ObjectNode) jsonNode;
-
- CypherQueryResponse cypherQueryResponse = mapper.convertValue(root, CypherQueryResponse.class);
-
- for ( List<List<ItemMetadata>> dataWrapper : cypherQueryResponse.getData()) {
-
- for (List<ItemMetadata> itemMetadatas : dataWrapper) {
-
- for (ItemMetadata itemMetadata : itemMetadatas) {
-
- ItemData itemData = itemMetadata.getData();
-
- LOGGER.debug("itemData: " + itemData);
-
- results.add(PropertyUtil.getInstance(mapper).unflattenMap(itemData.getAdditionalProperties()));
- }
-
- }
-
- }
- return results;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- super.prepare(configurationObject);
- mapper = StreamsJacksonMapper.getInstance();
-
- queryGraphHelper = new CypherQueryGraphHelper();
- httpGraphHelper = new Neo4jHttpGraphHelper();
-
- Objects.requireNonNull(queryGraphHelper);
- Objects.requireNonNull(httpGraphHelper);
- }
-
- @Override
- public StreamsResultSet readAll() {
- return readCurrent();
- }
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/http/Neo4jHttpPersistWriter.java b/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/http/Neo4jHttpPersistWriter.java
deleted file mode 100644
index e4b34bb..0000000
--- a/streams-contrib/streams-persist-neo4j/src/main/java/org/apache/streams/neo4j/http/Neo4jHttpPersistWriter.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.neo4j.http;
-
-import org.apache.streams.components.http.HttpPersistWriterConfiguration;
-import org.apache.streams.components.http.persist.SimpleHTTPPostPersistWriter;
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.graph.HttpGraphHelper;
-import org.apache.streams.graph.QueryGraphHelper;
-import org.apache.streams.neo4j.CypherQueryGraphHelper;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.neo4j.Neo4jConfiguration;
-import org.apache.streams.neo4j.Neo4jPersistUtil;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.util.EntityUtils;
-import org.javatuples.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * Adds activityobjects as vertices and activities as edges to a graph database with
- * an http rest endpoint (such as neo4j).
- */
-public class Neo4jHttpPersistWriter extends SimpleHTTPPostPersistWriter {
-
- public static final String STREAMS_ID = Neo4jHttpPersistWriter.class.getCanonicalName();
-
- private static final Logger LOGGER = LoggerFactory.getLogger(Neo4jHttpPersistWriter.class);
- private static final long MAX_WRITE_LATENCY = 1000;
-
- private Neo4jConfiguration configuration;
-
- private QueryGraphHelper queryGraphHelper;
- private HttpGraphHelper httpGraphHelper;
-
- private static ObjectMapper mapper;
-
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- /**
- * GraphHttpPersistWriter constructor - resolve GraphHttpConfiguration from JVM 'graph'.
- */
- public Neo4jHttpPersistWriter() {
- this(new ComponentConfigurator<>(Neo4jConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig().getConfig("neo4j")));
- }
-
- /**
- * GraphHttpPersistWriter constructor - use supplied GraphHttpConfiguration.
- * @param configuration GraphHttpConfiguration
- */
- public Neo4jHttpPersistWriter(Neo4jConfiguration configuration) {
- super((HttpPersistWriterConfiguration)StreamsJacksonMapper.getInstance().convertValue(configuration, HttpPersistWriterConfiguration.class).withHostname(configuration.getHosts().get(0)));
- super.configuration.setResourcePath("/db/data/transaction/commit/");
- this.configuration = configuration;
- }
-
- @Override
- protected ObjectNode preparePayload(StreamsDatum entry) throws Exception {
-
- List<Pair<String, Map<String, Object>>> statements = Neo4jPersistUtil.prepareStatements(entry);
-
- ObjectNode requestNode = mapper.createObjectNode();
- ArrayNode statementsArray = mapper.createArrayNode();
-
- for( Pair<String, Map<String, Object>> statement : statements ) {
- statementsArray.add(httpGraphHelper.writeData(statement));
- }
-
- requestNode.put("statements", statementsArray);
- return requestNode;
-
- }
-
- @Override
- protected ObjectNode executePost(HttpPost httpPost) {
-
- Objects.requireNonNull(httpPost);
-
- ObjectNode result = null;
-
- CloseableHttpResponse response = null;
-
- String entityString = null;
- try {
- response = httpclient.execute(httpPost);
- HttpEntity entity = response.getEntity();
- if (response.getStatusLine().getStatusCode() == 200 || response.getStatusLine().getStatusCode() == 201 && entity != null) {
- entityString = EntityUtils.toString(entity);
- result = mapper.readValue(entityString, ObjectNode.class);
- }
- LOGGER.debug("Writer response:\n{}\n{}\n{}", httpPost.toString(), response.getStatusLine().getStatusCode(), entityString);
- if ( result == null
- || (
- result.get("errors") != null
- && result.get("errors").isArray()
- && result.get("errors").iterator().hasNext()
- )
- ) {
- LOGGER.error("Write Error: " + result.get("errors"));
- } else {
- LOGGER.debug("Write Success");
- }
- } catch (IOException ex) {
- LOGGER.error("IO error:\n{}\n{}\n{}", httpPost.toString(), response, ex.getMessage());
- } catch (Exception ex) {
- LOGGER.error("Write Exception:\n{}\n{}\n{}", httpPost.toString(), response, ex.getMessage());
- } finally {
- try {
- if ( response != null) {
- response.close();
- }
- } catch (IOException ignored) {
- LOGGER.trace("ignored IOException", ignored);
- }
- }
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- super.prepare(null);
- mapper = StreamsJacksonMapper.getInstance();
-
- queryGraphHelper = new CypherQueryGraphHelper();
- httpGraphHelper = new Neo4jHttpGraphHelper();
-
- Objects.requireNonNull(queryGraphHelper);
- Objects.requireNonNull(httpGraphHelper);
- }
-
- @Override
- public void cleanUp() {
-
- LOGGER.info("exiting");
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/main/jsonschema/org/apache/streams/neo4j/CypherQueryResponse.json b/streams-contrib/streams-persist-neo4j/src/main/jsonschema/org/apache/streams/neo4j/CypherQueryResponse.json
deleted file mode 100644
index 4e80eb4..0000000
--- a/streams-contrib/streams-persist-neo4j/src/main/jsonschema/org/apache/streams/neo4j/CypherQueryResponse.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.neo4j.CypherQueryResponse",
- "properties": {
- "columns": {
- "type": "array",
- "id": "http://jsonschema.net/columns",
- "required": false,
- "items": {
- "type": "string",
- "id": "http://jsonschema.net/columns/0",
- "required": false
- }
- },
- "data": {
- "type": "array",
- "required": false,
- "items": {
- "type": "array",
- "required": false,
- "items": {
- "type": "array",
- "required": false,
- "items": {
- "type": "object",
- "javaType" : "org.apache.streams.neo4j.ItemMetadata",
- "properties": {
- "data": {
- "type": "object",
- "javaType" : "org.apache.streams.neo4j.ItemData"
- }
- }
- }
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-neo4j/src/main/jsonschema/org/apache/streams/neo4j/Neo4jConfiguration.json b/streams-contrib/streams-persist-neo4j/src/main/jsonschema/org/apache/streams/neo4j/Neo4jConfiguration.json
deleted file mode 100644
index a825168..0000000
--- a/streams-contrib/streams-persist-neo4j/src/main/jsonschema/org/apache/streams/neo4j/Neo4jConfiguration.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.neo4j.Neo4jConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "scheme": {
- "type": "string"
- },
- "hosts": {
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "port": {
- "type": "integer"
- },
- "username": {
- "type": "string"
- },
- "password": {
- "type": "string"
- }
- }
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/main/jsonschema/org/apache/streams/neo4j/Neo4jReaderConfiguration.json b/streams-contrib/streams-persist-neo4j/src/main/jsonschema/org/apache/streams/neo4j/Neo4jReaderConfiguration.json
deleted file mode 100644
index c17231b..0000000
--- a/streams-contrib/streams-persist-neo4j/src/main/jsonschema/org/apache/streams/neo4j/Neo4jReaderConfiguration.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.neo4j.Neo4jReaderConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "extends" : {"$ref":"Neo4jConfiguration.json"},
- "properties": {
- "query": {
- "type": "string",
- "required": "true"
- },
- "params": {
- "type": "object"
- }
- }
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/test/java/org/apache/streams/neo4j/test/Neo4jBoltPersistIT.java b/streams-contrib/streams-persist-neo4j/src/test/java/org/apache/streams/neo4j/test/Neo4jBoltPersistIT.java
deleted file mode 100644
index c45d975..0000000
--- a/streams-contrib/streams-persist-neo4j/src/test/java/org/apache/streams/neo4j/test/Neo4jBoltPersistIT.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.neo4j.test;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.neo4j.Neo4jConfiguration;
-import org.apache.streams.neo4j.Neo4jReaderConfiguration;
-import org.apache.streams.neo4j.bolt.Neo4jBoltClient;
-import org.apache.streams.neo4j.bolt.Neo4jBoltPersistReader;
-import org.apache.streams.neo4j.bolt.Neo4jBoltPersistWriter;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-
-import org.apache.commons.io.IOUtils;
-import org.neo4j.driver.v1.Session;
-import org.neo4j.driver.v1.Transaction;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import static org.testng.Assert.assertTrue;
-
-/**
- * Integration test for Neo4jBoltPersist.
- *
- * Test that graph db responses can be converted to streams data.
- */
-public class Neo4jBoltPersistIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(Neo4jBoltPersistIT.class);
-
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private Neo4jBoltClient testClient;
-
- private Neo4jConfiguration testConfiguration;
-
- @BeforeClass
- public void prepareTest() throws IOException {
-
- Config reference = ConfigFactory.load();
- File conf = new File("target/test-classes/Neo4jBoltPersistIT.conf");
- assertTrue(conf.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(Neo4jConfiguration.class).detectConfiguration(typesafe, "neo4j");
- testClient = Neo4jBoltClient.getInstance(testConfiguration);
-
- Session session = testClient.client().session();
- Transaction transaction = session.beginTransaction();
- transaction.run("MATCH ()-[r]-() DELETE r");
- transaction.run("MATCH (n) DETACH DELETE n");
- transaction.success();
- session.close();
- }
-
- @Test
- public void testNeo4jBoltPersist() throws Exception {
-
- Neo4jBoltPersistWriter testPersistWriter = new Neo4jBoltPersistWriter(testConfiguration);
- testPersistWriter.prepare(testConfiguration);
-
- InputStream testActivityFolderStream = Neo4jBoltPersistIT.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- int count = 0;
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = Neo4jBoltPersistIT.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- if( activity.getActor() != null && activity.getActor().getId() == null && activity.getActor().getObjectType() != null) {
- activity.getActor().setId(activity.getActor().getObjectType());
- }
- if( activity.getObject() != null && activity.getObject().getId() == null && activity.getObject().getObjectType() != null) {
- activity.getObject().setId(activity.getObject().getObjectType());
- }
- if( activity.getTarget() != null && activity.getTarget().getId() == null && activity.getTarget().getObjectType() != null) {
- activity.getTarget().setId(activity.getTarget().getObjectType());
- }
- if( activity.getId() == null && activity.getVerb() != null) {
- activity.setId(activity.getVerb());
- }
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- testPersistWriter.write( datum );
- LOGGER.info("Wrote: " + activity.getVerb() );
- count++;
- }
-
- testPersistWriter.cleanUp();
-
- LOGGER.info("Total Written: {}", count );
- Assert.assertEquals(count, 89);
-
- Neo4jReaderConfiguration vertexReaderConfiguration= MAPPER.convertValue(testConfiguration, Neo4jReaderConfiguration.class);
- vertexReaderConfiguration.setQuery("MATCH (v) return v");
- Neo4jBoltPersistReader vertexReader = new Neo4jBoltPersistReader(vertexReaderConfiguration);
- vertexReader.prepare(null);
- StreamsResultSet vertexResultSet = vertexReader.readAll();
- LOGGER.info("Total Read: {}", vertexResultSet.size() );
- Assert.assertEquals(vertexResultSet.size(), 24);
-
- Neo4jReaderConfiguration edgeReaderConfiguration= MAPPER.convertValue(testConfiguration, Neo4jReaderConfiguration.class);
- edgeReaderConfiguration.setQuery("MATCH (s)-[r]->(d) return r");
- Neo4jBoltPersistReader edgeReader = new Neo4jBoltPersistReader(edgeReaderConfiguration);
- edgeReader.prepare(null);
- StreamsResultSet edgeResultSet = edgeReader.readAll();
- LOGGER.info("Total Read: {}", edgeResultSet.size() );
- Assert.assertEquals(edgeResultSet.size(), 100);
-
- }
-
- @AfterClass
- public void cleanup() throws Exception {
- Session session = testClient.client().session();
- Transaction transaction = session.beginTransaction();
- transaction.run("MATCH ()-[r]-() DELETE r");
- transaction.run("MATCH (n) DETACH DELETE n");
- transaction.success();
- session.close();
- }
-
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/test/java/org/apache/streams/neo4j/test/Neo4jHttpPersistIT.java b/streams-contrib/streams-persist-neo4j/src/test/java/org/apache/streams/neo4j/test/Neo4jHttpPersistIT.java
deleted file mode 100644
index a5b0d30..0000000
--- a/streams-contrib/streams-persist-neo4j/src/test/java/org/apache/streams/neo4j/test/Neo4jHttpPersistIT.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.neo4j.test;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.neo4j.Neo4jConfiguration;
-import org.apache.streams.neo4j.Neo4jReaderConfiguration;
-import org.apache.streams.neo4j.http.Neo4jHttpClient;
-import org.apache.streams.neo4j.http.Neo4jHttpPersistReader;
-import org.apache.streams.neo4j.http.Neo4jHttpPersistWriter;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import static org.testng.Assert.assertTrue;
-
-/**
- * Integration test for Neo4jHttpPersist.
- *
- * Test that graph db responses can be converted to streams data.
- */
-public class Neo4jHttpPersistIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(Neo4jHttpPersistIT.class);
-
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private Neo4jHttpClient testClient;
-
- private Neo4jConfiguration testConfiguration;
-
- @BeforeClass
- public void prepareTest() throws IOException {
-
- Config reference = ConfigFactory.load();
- File conf = new File("target/test-classes/Neo4jHttpPersistIT.conf");
- assertTrue(conf.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(Neo4jConfiguration.class).detectConfiguration(typesafe, "neo4j");
-
- }
-
- @Test
- public void testNeo4jHttpPersist() throws Exception {
-
- Neo4jHttpPersistWriter testPersistWriter = new Neo4jHttpPersistWriter(testConfiguration);
- testPersistWriter.prepare(null);
-
- InputStream testActivityFolderStream = Neo4jHttpPersistIT.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- // write data
-
- int count = 0;
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = Neo4jHttpPersistIT.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- if( activity.getActor() != null && activity.getActor().getId() == null && activity.getActor().getObjectType() != null) {
- activity.getActor().setId(activity.getActor().getObjectType());
- }
- if( activity.getObject() != null && activity.getObject().getId() == null && activity.getObject().getObjectType() != null) {
- activity.getObject().setId(activity.getObject().getObjectType());
- }
- if( activity.getTarget() != null && activity.getTarget().getId() == null && activity.getTarget().getObjectType() != null) {
- activity.getTarget().setId(activity.getTarget().getObjectType());
- }
- if( activity.getId() == null && activity.getVerb() != null) {
- activity.setId(activity.getVerb());
- }
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- testPersistWriter.write( datum );
- LOGGER.info("Wrote: " + activity.getVerb() );
- count++;
- }
-
- testPersistWriter.cleanUp();
-
- LOGGER.info("Total Written: {}", count );
- Assert.assertEquals(count, 89);
-
- Neo4jReaderConfiguration vertexReaderConfiguration= MAPPER.convertValue(testConfiguration, Neo4jReaderConfiguration.class);
- vertexReaderConfiguration.setQuery("MATCH (v) return v");
- Neo4jHttpPersistReader vertexReader = new Neo4jHttpPersistReader(vertexReaderConfiguration);
- vertexReader.prepare(null);
- StreamsResultSet vertexResultSet = vertexReader.readAll();
- LOGGER.info("Total Read: {}", vertexResultSet.size() );
- Assert.assertEquals(vertexResultSet.size(), 24);
-
- Neo4jReaderConfiguration edgeReaderConfiguration= MAPPER.convertValue(testConfiguration, Neo4jReaderConfiguration.class);
- edgeReaderConfiguration.setQuery("MATCH (s)-[r]->(d) return r");
- Neo4jHttpPersistReader edgeReader = new Neo4jHttpPersistReader(edgeReaderConfiguration);
- edgeReader.prepare(null);
- StreamsResultSet edgeResultSet = edgeReader.readAll();
- LOGGER.info("Total Read: {}", edgeResultSet.size() );
- Assert.assertEquals(edgeResultSet.size(), 100);
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/test/java/org/apache/streams/neo4j/test/TestCypherQueryGraphHelper.java b/streams-contrib/streams-persist-neo4j/src/test/java/org/apache/streams/neo4j/test/TestCypherQueryGraphHelper.java
deleted file mode 100644
index 12f3306..0000000
--- a/streams-contrib/streams-persist-neo4j/src/test/java/org/apache/streams/neo4j/test/TestCypherQueryGraphHelper.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.neo4j.test;
-
-import org.apache.streams.neo4j.CypherQueryGraphHelper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import org.javatuples.Pair;
-import org.junit.Test;
-
-import java.util.Map;
-
-/**
- * TestCypherQueryGraphHelper tests
- * @see org.apache.streams.graph.neo4j.CypherQueryGraphHelper
- */
-public class TestCypherQueryGraphHelper {
-
- CypherQueryGraphHelper helper = new CypherQueryGraphHelper();
-
- @Test
- public void getVertexRequestIdTest() throws Exception {
-
- Pair<String, Map<String, Object>> queryAndParams = helper.getVertexRequest("id");
- assert(queryAndParams != null);
- assert(queryAndParams.getValue0() != null);
-
- }
-
- @Test
- public void getVertexRequestLongTest() throws Exception {
-
- Pair<String, Map<String, Object>> queryAndParams = helper.getVertexRequest(new Long(1));
-
- assert(queryAndParams != null);
- assert(queryAndParams.getValue0() != null);
-
- }
-
- @Test
- public void createVertexRequestTest() throws Exception {
-
- ActivityObject activityObject = new ActivityObject();
- activityObject.setId("id");
- activityObject.setObjectType("type");
- activityObject.setContent("content");
-
- Pair<String, Map<String, Object>> queryAndParams = helper.createVertexRequest(activityObject);
- assert(queryAndParams != null);
- assert(queryAndParams.getValue0() != null);
- assert(queryAndParams.getValue1() != null);
-
- }
-
- @Test
- public void mergeVertexRequestTest() throws Exception {
-
- ActivityObject activityObject = new ActivityObject();
- activityObject.setId("id");
- activityObject.setObjectType("type");
- activityObject.setContent("content");
-
- Pair<String, Map<String, Object>> queryAndParams = helper.mergeVertexRequest(activityObject);
- assert(queryAndParams != null);
- assert(queryAndParams.getValue0() != null);
- assert(queryAndParams.getValue1() != null);
-
- }
-
- @Test
- public void createActorObjectEdgeRequestTest() throws Exception {
-
- ActivityObject actor = new ActivityObject();
- actor.setId("actor");
- actor.setObjectType("type");
- actor.setContent("content");
-
- ActivityObject object = new ActivityObject();
- object.setId("object");
- object.setObjectType("type");
- object.setContent("content");
-
- Activity activity = new Activity();
- activity.setId("activity");
- activity.setVerb("verb");
- activity.setContent("content");
-
- activity.setActor(actor);
- activity.setObject(object);
-
- Pair<String, Map<String, Object>> queryAndParams = helper.createActorObjectEdge(activity);
-
- assert(queryAndParams != null);
- assert(queryAndParams.getValue0() != null);
- assert(queryAndParams.getValue1() != null);
-
- }
-
- @Test
- public void createEdgeRequestTest() throws Exception {
-
- ActivityObject actor = new ActivityObject();
- actor.setId("actor");
- actor.setObjectType("type");
- actor.setContent("content");
-
- ActivityObject object = new ActivityObject();
- object.setId("object");
- object.setObjectType("type");
- object.setContent("content");
-
- ActivityObject target = new ActivityObject();
- object.setId("target");
- object.setObjectType("type");
-
- Activity activity = new Activity();
- activity.setId("activity");
- activity.setVerb("verb");
- activity.setContent("content");
-
- activity.setActor(actor);
- activity.setObject(object);
- activity.setObject(target);
-
- Pair<String, Map<String, Object>> queryAndParams = helper.createActorTargetEdge(activity);
-
- assert(queryAndParams != null);
- assert(queryAndParams.getValue0() != null);
- assert(queryAndParams.getValue1() != null);
-
- }
-}
diff --git a/streams-contrib/streams-persist-neo4j/src/test/resources/Neo4jBoltPersistIT.conf b/streams-contrib/streams-persist-neo4j/src/test/resources/Neo4jBoltPersistIT.conf
deleted file mode 100644
index 1d5ec35..0000000
--- a/streams-contrib/streams-persist-neo4j/src/test/resources/Neo4jBoltPersistIT.conf
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-neo4j {
- scheme = "tcp"
- hosts += ${neo4j.tcp.host}
- port = ${neo4j.tcp.port}
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-neo4j/src/test/resources/Neo4jHttpPersistIT.conf b/streams-contrib/streams-persist-neo4j/src/test/resources/Neo4jHttpPersistIT.conf
deleted file mode 100644
index 929b3ed..0000000
--- a/streams-contrib/streams-persist-neo4j/src/test/resources/Neo4jHttpPersistIT.conf
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-neo4j {
- scheme = "http"
- hosts += ${neo4j.http.host}
- port = ${neo4j.http.port}
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-riak/pom.xml b/streams-contrib/streams-persist-riak/pom.xml
deleted file mode 100644
index 2a84b75..0000000
--- a/streams-contrib/streams-persist-riak/pom.xml
+++ /dev/null
@@ -1,229 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-persist-riak</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Riak Module</description>
-
- <properties>
- <riak.version>2.1.1</riak.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>com.basho.riak</groupId>
- <artifactId>riak-client</artifactId>
- <version>${riak.version}</version>
- <optional>true</optional>
- </dependency>
- </dependencies>
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.riak.pojo</targetPackage>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-http</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <profile>
- <id>dockerITs</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>skipITs</name>
- <value>false</value>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>io.fabric8</groupId>
- <artifactId>docker-maven-plugin</artifactId>
- <configuration combine.self="override">
- <watchInterval>500</watchInterval>
- <logDate>default</logDate>
- <verbose>true</verbose>
- <autoPull>on</autoPull>
- <images>
- <image>
- <name>lapax/riak</name>
- <alias>riak</alias>
- <run>
- <env>
- <NEO4J_AUTH>none</NEO4J_AUTH>
- </env>
- <namingStrategy>none</namingStrategy>
- <ports>
- <port>${riak.http.host}:${riak.http.port}:8098</port>
- <port>${riak.tcp.host}:${riak.tcp.port}:8087</port>
- </ports>
- <portPropertyFile>riak.properties</portPropertyFile>
- <wait>
- <log>riak startup</log>
- <http>
- <url>http://${riak.http.host}:${riak.http.port}</url>
- <method>GET</method>
- <status>200</status>
- </http>
- <time>20000</time>
- <kill>1000</kill>
- <shutdown>500</shutdown>
- </wait>
- <log>
- <enabled>true</enabled>
- <date>default</date>
- <color>cyan</color>
- </log>
- </run>
- <watch>
- <mode>none</mode>
- </watch>
- </image>
-
- </images>
- </configuration>
-
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeTypes>test-jar</includeTypes>
- </configuration>
- <executions>
- <execution>
- <id>test-resource-dependencies</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- </profile>
- </profiles>
-
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/binary/RiakBinaryClient.java b/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/binary/RiakBinaryClient.java
deleted file mode 100644
index 214eed9..0000000
--- a/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/binary/RiakBinaryClient.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.riak.binary;
-
-import org.apache.streams.riak.pojo.RiakConfiguration;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * RiakBinaryClient maintains shared connections to riak via binary protocol.
- */
-public class RiakBinaryClient {
-
- private static final Logger LOGGER = LoggerFactory
- .getLogger(RiakBinaryClient.class);
-
- private com.basho.riak.client.api.RiakClient client;
-
- public RiakConfiguration config;
-
- private RiakBinaryClient(RiakConfiguration config) {
- this.config = config;
- try {
- this.start();
- } catch (Exception e) {
- e.printStackTrace();
- this.client = null;
- }
- }
-
- private static Map<RiakConfiguration, RiakBinaryClient> INSTANCE_MAP = new ConcurrentHashMap<>();
-
- public static RiakBinaryClient getInstance(RiakConfiguration riakConfiguration) {
- if ( INSTANCE_MAP != null
- && INSTANCE_MAP.size() > 0
- && INSTANCE_MAP.containsKey(riakConfiguration)) {
- return INSTANCE_MAP.get(riakConfiguration);
- } else {
- RiakBinaryClient instance = new RiakBinaryClient(riakConfiguration);
- if( instance != null && instance.client != null ) {
- INSTANCE_MAP.put(riakConfiguration, instance);
- return instance;
- } else {
- return null;
- }
- }
- }
-
- public void start() throws Exception {
-
- Objects.nonNull(config);
-
- LOGGER.info("RiakHttpClient.start {}", config);
-
- this.client = com.basho.riak.client.api.RiakClient.newClient(config.getPort().intValue(), config.getHosts());
-
- Objects.nonNull(client);
-
- Objects.nonNull(client.getRiakCluster());
-
- assert( client.getRiakCluster().getNodes().size() > 0 );
- }
-
- public void stop() throws Exception {
- this.client = null;
- }
-
- public RiakConfiguration config() {
- return config;
- }
-
- public com.basho.riak.client.api.RiakClient client() {
- return client;
- }
-
-}
diff --git a/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/binary/RiakBinaryPersistReader.java b/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/binary/RiakBinaryPersistReader.java
deleted file mode 100644
index 3846d53..0000000
--- a/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/binary/RiakBinaryPersistReader.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.riak.binary;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.riak.pojo.RiakConfiguration;
-
-import com.basho.riak.client.api.commands.kv.FetchValue;
-import com.basho.riak.client.api.commands.kv.ListKeys;
-import com.basho.riak.client.api.commands.kv.MultiFetch;
-import com.basho.riak.client.core.RiakFuture;
-import com.basho.riak.client.core.query.Location;
-import com.basho.riak.client.core.query.Namespace;
-import com.google.common.collect.Queues;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-
-import java.math.BigInteger;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.LinkedBlockingQueue;
-
-/**
- * RiakBinaryPersistReader reads documents from riak via binary protocol.
- */
-public class RiakBinaryPersistReader implements StreamsPersistReader {
-
- private RiakConfiguration configuration;
- public RiakBinaryClient client;
-
- public RiakBinaryPersistReader(RiakConfiguration configuration) {
- this.configuration = configuration;
- }
-
- @Override
- public String getId() {
- return "RiakBinaryPersistReader";
- }
-
- @Override
- public void prepare(Object configurationObject) {
- client = RiakBinaryClient.getInstance(this.configuration);
- }
-
- @Override
- public void cleanUp() {
- client = null;
- }
-
- @Override
- public synchronized StreamsResultSet readAll() {
-
- Queue<StreamsDatum> readAllQueue = constructQueue();
-
- Namespace ns = new Namespace(configuration.getDefaultBucketType(), configuration.getDefaultBucket());
-
- ListKeys lk = new ListKeys.Builder(ns).build();
-
- ListKeys.Response listKeysResponse = null;
- try {
- listKeysResponse = client.client().execute(lk);
- } catch (Exception e) {
- e.printStackTrace();
- return null;
- }
-
- MultiFetch multiFetch = new MultiFetch.Builder().addLocations(listKeysResponse).build();
- MultiFetch.Response multiFetchResponse = null;
- try {
- multiFetchResponse = client.client().execute(multiFetch);
- } catch (ExecutionException e) {
- e.printStackTrace();
- return null;
- } catch (InterruptedException e) {
- e.printStackTrace();
- return null;
- }
-
- for (RiakFuture<FetchValue.Response, Location> f : multiFetchResponse) {
- try {
- FetchValue.Response response = f.get();
- readAllQueue.add(new StreamsDatum(response.getValue(String.class), f.getQueryInfo().getKeyAsString()));
- }
- catch (ExecutionException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
-
- return new StreamsResultSet(readAllQueue);
- }
-
- @Override
- public void startStream() {
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean isRunning() {
- return Objects.nonNull(client);
- }
-
- private Queue<StreamsDatum> constructQueue() {
- return Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
- }
-}
diff --git a/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/binary/RiakBinaryPersistWriter.java b/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/binary/RiakBinaryPersistWriter.java
deleted file mode 100644
index eb7b0bf..0000000
--- a/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/binary/RiakBinaryPersistWriter.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.riak.binary;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.riak.pojo.RiakConfiguration;
-
-import com.basho.riak.client.api.commands.kv.StoreValue;
-import com.basho.riak.client.core.query.Location;
-import com.basho.riak.client.core.query.Namespace;
-import com.basho.riak.client.core.query.RiakObject;
-import com.basho.riak.client.core.util.BinaryValue;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Objects;
-import java.util.concurrent.ExecutionException;
-
-/**
- * RiakBinaryPersistWriter writes documents to riak via binary protocol.
- */
-public class RiakBinaryPersistWriter implements StreamsPersistWriter {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RiakBinaryPersistWriter.class);
-
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private RiakConfiguration configuration;
- private RiakBinaryClient client;
-
- public RiakBinaryPersistWriter(RiakConfiguration configuration) {
- this.configuration = configuration;
- }
-
- @Override
- public String getId() {
- return "RiakBinaryPersistWriter";
- }
-
- @Override
- public void prepare(Object configurationObject) {
- client = RiakBinaryClient.getInstance(this.configuration);
- }
-
- @Override
- public void cleanUp() {
- client = null;
- }
-
- @Override
- public void write(StreamsDatum entry) {
-
- Objects.nonNull(client);
-
- String id = null;
- String document;
- String bucket;
- String bucketType;
- String contentType;
- String charset;
- if( StringUtils.isNotBlank(entry.getId())) {
- id = entry.getId();
- }
- if( entry.getDocument() instanceof String) {
- document = (String)entry.getDocument();
- } else {
- try {
- document = MAPPER.writeValueAsString(entry.getDocument());
- } catch( Exception e ) {
- LOGGER.warn("Exception", e);
- return;
- }
- }
- if( entry.getMetadata() != null
- && entry.getMetadata().containsKey("bucket")
- && entry.getMetadata().get("bucket") instanceof String
- && StringUtils.isNotBlank((String)entry.getMetadata().get("bucket") )) {
- bucket = (String)entry.getMetadata().get("bucket");
- } else {
- bucket = configuration.getDefaultBucket();
- }
- if( entry.getMetadata() != null
- && entry.getMetadata().containsKey("bucketType")
- && entry.getMetadata().get("bucketType") instanceof String
- && StringUtils.isNotBlank((String)entry.getMetadata().get("bucketType") )) {
- bucketType = (String)entry.getMetadata().get("bucketType");
- } else {
- bucketType = configuration.getDefaultBucketType();
- }
- if( entry.getMetadata() != null
- && entry.getMetadata().containsKey("charset")
- && entry.getMetadata().get("charset") instanceof String
- && StringUtils.isNotBlank((String)entry.getMetadata().get("charset") )) {
- charset = (String)entry.getMetadata().get("charset");
- } else {
- charset = configuration.getDefaultCharset();
- }
- if( entry.getMetadata() != null
- && entry.getMetadata().containsKey("contentType")
- && entry.getMetadata().get("contentType") instanceof String
- && StringUtils.isNotBlank((String)entry.getMetadata().get("contentType") )) {
- contentType = (String)entry.getMetadata().get("contentType");
- } else {
- contentType = configuration.getDefaultContentType();
- }
-
- try {
-
- RiakObject riakObject = new RiakObject();
- riakObject.setContentType(contentType);
- riakObject.setCharset(charset);
- riakObject.setValue(BinaryValue.create(document));
-
- Namespace ns = new Namespace(bucketType, bucket);
- StoreValue.Builder storeValueBuilder = new StoreValue.Builder(riakObject);
-
- if( id != null && StringUtils.isNotBlank(id)) {
- Location location = new Location(ns, id);
- storeValueBuilder = storeValueBuilder.withLocation(location);
- } else {
- storeValueBuilder = storeValueBuilder.withNamespace(ns);
- }
-
- StoreValue store = storeValueBuilder.build();
-
- StoreValue.Response storeResponse = client.client().execute(store);
-
- LOGGER.debug("storeResponse", storeResponse);
-
- } catch (InterruptedException e) {
- e.printStackTrace();
- } catch (ExecutionException e) {
- e.printStackTrace();
- }
-
- }
-
-
-}
diff --git a/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/http/RiakHttpClient.java b/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/http/RiakHttpClient.java
deleted file mode 100644
index 7155b52..0000000
--- a/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/http/RiakHttpClient.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.riak.http;
-
-import org.apache.streams.riak.pojo.RiakConfiguration;
-
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URI;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * RiakHttpClient maintains shared connections to riak via http.
- */
-public class RiakHttpClient {
-
- private static final Logger LOGGER = LoggerFactory
- .getLogger(RiakHttpClient.class);
-
- public RiakConfiguration config;
-
- protected CloseableHttpClient client;
- protected URI baseURI;
-
- private RiakHttpClient(RiakConfiguration config) {
- this.config = config;
- try {
- this.start();
- } catch (Exception e) {
- e.printStackTrace();
- this.client = null;
- }
- }
-
- private static Map<RiakConfiguration, RiakHttpClient> INSTANCE_MAP = new ConcurrentHashMap<>();
-
- public static RiakHttpClient getInstance(RiakConfiguration riakConfiguration) {
- if ( INSTANCE_MAP != null
- && INSTANCE_MAP.size() > 0
- && INSTANCE_MAP.containsKey(riakConfiguration)) {
- return INSTANCE_MAP.get(riakConfiguration);
- } else {
- RiakHttpClient instance = new RiakHttpClient(riakConfiguration);
- if( instance != null && instance.client != null ) {
- INSTANCE_MAP.put(riakConfiguration, instance);
- return instance;
- } else {
- return null;
- }
- }
- }
-
- public void start() throws Exception {
- Objects.nonNull(config);
- assert(config.getScheme().startsWith("http"));
- URIBuilder uriBuilder = new URIBuilder();
- uriBuilder.setScheme(config.getScheme());
- uriBuilder.setHost(config.getHosts().get(0));
- uriBuilder.setPort(config.getPort().intValue());
- baseURI = uriBuilder.build();
- client = HttpClients.createDefault();
- }
-
- public void stop() {
- try {
- client.close();
- } catch( Exception e) {
- LOGGER.error( "Exception", e );
- } finally {
- client = null;
- }
- }
-
- public RiakConfiguration config() {
- return config;
- }
-
- public HttpClient client() {
- return client;
- }
-
-}
diff --git a/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/http/RiakHttpPersistReader.java b/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/http/RiakHttpPersistReader.java
deleted file mode 100644
index 353bbbf..0000000
--- a/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/http/RiakHttpPersistReader.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.riak.http;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistReader;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.riak.pojo.RiakConfiguration;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.google.common.collect.Queues;
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.ClientProtocolException;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.util.EntityUtils;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.math.BigInteger;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Iterator;
-import java.util.Queue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-/**
- * RiakHttpPersistReader reads documents from riak via http.
- */
-public class RiakHttpPersistReader implements StreamsPersistReader {
-
- private RiakConfiguration configuration;
- private RiakHttpClient client;
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RiakHttpPersistReader.class);
-
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- public RiakHttpPersistReader(RiakConfiguration configuration) {
- this.configuration = configuration;
- }
-
- @Override
- public String getId() {
- return "RiakHttpPersistReader";
- }
-
- @Override
- public void prepare(Object configurationObject) {
- client = RiakHttpClient.getInstance(this.configuration);
- }
-
- @Override
- public void cleanUp() {
- client = null;
- }
-
- @Override
- public StreamsResultSet readAll() {
-
- Queue<StreamsDatum> readAllQueue = constructQueue();
-
- URIBuilder lk = null;
-
- try {
-
- lk = new URIBuilder(client.baseURI.toString());
- lk.setPath(client.baseURI.getPath().concat("/buckets/"+configuration.getDefaultBucket()+"/keys"));
- lk.setParameter("keys", "true");
-
- } catch (URISyntaxException e) {
- LOGGER.warn("URISyntaxException", e);
- }
-
- HttpResponse lkResponse = null;
- try {
- HttpGet lkGet = new HttpGet(lk.build());
- lkResponse = client.client().execute(lkGet);
- } catch (IOException e) {
- LOGGER.warn("IOException", e);
- return null;
- } catch (URISyntaxException e) {
- LOGGER.warn("URISyntaxException", e);
- return null;
- }
-
- String lkEntityString = null;
- try {
- lkEntityString = EntityUtils.toString(lkResponse.getEntity());
- } catch (IOException e) {
- LOGGER.warn("IOException", e);
- return null;
- }
-
- JsonNode lkEntityNode = null;
- try {
- lkEntityNode = MAPPER.readValue(lkEntityString, JsonNode.class);
- } catch (IOException e) {
- LOGGER.warn("IOException", e);
- return null;
- }
-
- ArrayNode keysArray = null;
- keysArray = (ArrayNode) lkEntityNode.get("keys");
- Iterator<JsonNode> keysIterator = keysArray.iterator();
-
- while( keysIterator.hasNext()) {
- JsonNode keyNode = keysIterator.next();
- String key = keyNode.asText();
-
- URIBuilder gk = null;
-
- try {
-
- gk = new URIBuilder(client.baseURI.toString());
- gk.setPath(client.baseURI.getPath().concat("/buckets/"+configuration.getDefaultBucket()+"/keys/"+key));
-
- } catch (URISyntaxException e) {
- LOGGER.warn("URISyntaxException", e);
- continue;
- }
-
- HttpResponse gkResponse = null;
- try {
- HttpGet gkGet = new HttpGet(gk.build());
- gkResponse = client.client().execute(gkGet);
- } catch (IOException e) {
- LOGGER.warn("IOException", e);
- continue;
- } catch (URISyntaxException e) {
- LOGGER.warn("URISyntaxException", e);
- continue;
- }
-
- String gkEntityString = null;
- try {
- gkEntityString = EntityUtils.toString(gkResponse.getEntity());
- } catch (IOException e) {
- LOGGER.warn("IOException", e);
- continue;
- }
-
- readAllQueue.add(new StreamsDatum(gkEntityString, key));
- }
-
- return new StreamsResultSet(readAllQueue);
- }
-
- @Override
- public void startStream() {
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean isRunning() {
- return false;
- }
-
- private Queue<StreamsDatum> constructQueue() {
- return Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
- }
-}
diff --git a/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/http/RiakHttpPersistWriter.java b/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/http/RiakHttpPersistWriter.java
deleted file mode 100644
index b51b053..0000000
--- a/streams-contrib/streams-persist-riak/src/main/java/org/apache/streams/riak/http/RiakHttpPersistWriter.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.riak.http;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.riak.pojo.RiakConfiguration;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.StringEntity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Objects;
-
-/**
- * RiakHttpPersistWriter writes documents to riak via http.
- */
-public class RiakHttpPersistWriter implements StreamsPersistWriter {
-
- private RiakConfiguration configuration;
- private RiakHttpClient client;
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RiakHttpPersistWriter.class);
-
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- public RiakHttpPersistWriter(RiakConfiguration configuration) {
- this.configuration = configuration;
- }
-
- @Override
- public String getId() {
- return "RiakHttpPersistWriter";
- }
-
- @Override
- public void prepare(Object configurationObject) {
- client = RiakHttpClient.getInstance(this.configuration);
- }
-
- @Override
- public void cleanUp() {
- client = null;
- }
-
- @Override
- public void write(StreamsDatum entry) {
-
- Objects.nonNull(client);
-
- String id = null;
- String document;
- String bucket;
- String bucketType;
- String contentType;
- String charset;
- if( StringUtils.isNotBlank(entry.getId())) {
- id = entry.getId();
- }
- if( entry.getDocument() instanceof String) {
- document = (String)entry.getDocument();
- } else {
- try {
- document = MAPPER.writeValueAsString(entry.getDocument());
- } catch( Exception e ) {
- LOGGER.warn("Exception", e);
- return;
- }
- }
- if( entry.getMetadata() != null
- && entry.getMetadata().containsKey("bucket")
- && entry.getMetadata().get("bucket") instanceof String
- && StringUtils.isNotBlank((String)entry.getMetadata().get("bucket") )) {
- bucket = (String)entry.getMetadata().get("bucket");
- } else {
- bucket = configuration.getDefaultBucket();
- }
- if( entry.getMetadata() != null
- && entry.getMetadata().containsKey("bucketType")
- && entry.getMetadata().get("bucketType") instanceof String
- && StringUtils.isNotBlank((String)entry.getMetadata().get("bucketType") )) {
- bucketType = (String)entry.getMetadata().get("bucketType");
- } else {
- bucketType = configuration.getDefaultBucketType();
- }
- if( entry.getMetadata() != null
- && entry.getMetadata().containsKey("charset")
- && entry.getMetadata().get("charset") instanceof String
- && StringUtils.isNotBlank((String)entry.getMetadata().get("charset") )) {
- charset = (String)entry.getMetadata().get("charset");
- } else {
- charset = configuration.getDefaultCharset();
- }
- if( entry.getMetadata() != null
- && entry.getMetadata().containsKey("contentType")
- && entry.getMetadata().get("contentType") instanceof String
- && StringUtils.isNotBlank((String)entry.getMetadata().get("contentType") )) {
- contentType = (String)entry.getMetadata().get("contentType");
- } else {
- contentType = configuration.getDefaultContentType();
- }
-
- URIBuilder uriBuilder = new URIBuilder(client.baseURI);
- if( bucket != null && StringUtils.isNotBlank(bucket)) {
- uriBuilder.setPath("/riak/"+bucket);
- }
- if( id != null && StringUtils.isNotBlank(id)) {
- uriBuilder.setPath("/riak/"+bucket+"/"+id);
- }
-
- URI uri;
- try {
- uri = uriBuilder.build();
- } catch (URISyntaxException e) {
- LOGGER.warn("URISyntaxException", e);
- return;
- }
-
- HttpPost post = new HttpPost();
- post.setHeader("Content-Type", contentType + "; charset=" + charset);
- post.setURI(uri);
- HttpEntity entity;
- try {
- entity = new StringEntity(document);
- post.setEntity(entity);
- } catch (UnsupportedEncodingException e) {
- LOGGER.warn("UnsupportedEncodingException", e);
- return;
- }
-
- try {
- HttpResponse response = client.client().execute(post);
- } catch (IOException e) {
- LOGGER.warn("IOException", e);
- return;
- }
-
- }
-}
diff --git a/streams-contrib/streams-persist-riak/src/main/jsonschema/RiakConfiguration.json b/streams-contrib/streams-persist-riak/src/main/jsonschema/RiakConfiguration.json
deleted file mode 100644
index 917347f..0000000
--- a/streams-contrib/streams-persist-riak/src/main/jsonschema/RiakConfiguration.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.riak.pojo.RiakConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "scheme": {
- "type": "string"
- },
- "hosts": {
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "port": {
- "type": "integer"
- },
- "defaultBucket": {
- "type": "string",
- "default": "streams"
- },
- "defaultBucketType": {
- "type": "string",
- "default": "default"
- },
- "defaultCharset": {
- "type": "string",
- "default": "UTF-8"
- },
- "defaultContentType": {
- "type": "string",
- "default": "application/json"
- }
- }
-}
diff --git a/streams-contrib/streams-persist-riak/src/test/java/org/apache/streams/riak/test/RiakBinaryPersistIT.java b/streams-contrib/streams-persist-riak/src/test/java/org/apache/streams/riak/test/RiakBinaryPersistIT.java
deleted file mode 100644
index 523ec01..0000000
--- a/streams-contrib/streams-persist-riak/src/test/java/org/apache/streams/riak/test/RiakBinaryPersistIT.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.riak.test;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.riak.binary.RiakBinaryClient;
-import org.apache.streams.riak.binary.RiakBinaryPersistReader;
-import org.apache.streams.riak.binary.RiakBinaryPersistWriter;
-import org.apache.streams.riak.pojo.RiakConfiguration;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import static org.testng.Assert.assertTrue;
-
-/**
- * Integration test for RiakBinaryPersist.
- */
-public class RiakBinaryPersistIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RiakBinaryPersistIT.class);
-
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private RiakBinaryClient testClient;
-
- private RiakConfiguration testConfiguration;
-
- @BeforeClass
- public void prepareTest() throws IOException {
-
- Config reference = ConfigFactory.load();
- File conf = new File("target/test-classes/RiakBinaryPersistIT.conf");
- assertTrue(conf.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(RiakConfiguration.class).detectConfiguration(typesafe, "riak");
- testClient = RiakBinaryClient.getInstance(testConfiguration);
-
- }
-
- @Test
- public void testRiakBinaryPersist() throws Exception {
-
- RiakBinaryPersistWriter testPersistWriter = new RiakBinaryPersistWriter(testConfiguration);
- testPersistWriter.prepare(testConfiguration);
-
- InputStream testActivityFolderStream = RiakBinaryPersistIT.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- int count = 0;
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = RiakBinaryPersistIT.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
-
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- testPersistWriter.write( datum );
- LOGGER.info("Wrote: " + activity.getVerb() );
- count++;
- }
-
- testPersistWriter.cleanUp();
-
- LOGGER.info("Total Written: {}", count );
- Assert.assertEquals(count, 89);
-
- RiakBinaryPersistReader testPersistReader = new RiakBinaryPersistReader(testConfiguration);
- testPersistReader.prepare(testConfiguration);
-
- StreamsResultSet readerResultSet = testPersistReader.readAll();
- LOGGER.info("Total Read: {}", readerResultSet.size() );
- Assert.assertEquals(readerResultSet.size(), 89);
-
- }
-
- @AfterClass
- public void cleanup() throws Exception {
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-riak/src/test/java/org/apache/streams/riak/test/RiakHttpPersistIT.java b/streams-contrib/streams-persist-riak/src/test/java/org/apache/streams/riak/test/RiakHttpPersistIT.java
deleted file mode 100644
index 2eeea30..0000000
--- a/streams-contrib/streams-persist-riak/src/test/java/org/apache/streams/riak/test/RiakHttpPersistIT.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.riak.test;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.riak.http.RiakHttpClient;
-import org.apache.streams.riak.http.RiakHttpPersistReader;
-import org.apache.streams.riak.http.RiakHttpPersistWriter;
-import org.apache.streams.riak.pojo.RiakConfiguration;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import static org.testng.Assert.assertTrue;
-
-/**
- * Integration test for RiakHttpPersist.
- */
-public class RiakHttpPersistIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RiakHttpPersistIT.class);
-
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private RiakHttpClient testClient;
-
- private RiakConfiguration testConfiguration;
-
- @BeforeClass
- public void prepareTest() throws IOException {
-
- Config reference = ConfigFactory.load();
- File conf = new File("target/test-classes/RiakHttpPersistIT.conf");
- assertTrue(conf.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf, ConfigParseOptions.defaults().setAllowMissing(false));
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
- testConfiguration = new ComponentConfigurator<>(RiakConfiguration.class).detectConfiguration(typesafe, "riak");
- testClient = RiakHttpClient.getInstance(testConfiguration);
-
- }
-
- @Test
- public void testRiakHttpPersist() throws Exception {
-
- RiakHttpPersistWriter testPersistWriter = new RiakHttpPersistWriter(testConfiguration);
- testPersistWriter.prepare(testConfiguration);
-
- InputStream testActivityFolderStream = RiakHttpPersistIT.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- // write data
-
- int count = 0;
- for( String file : files) {
- LOGGER.info("File: " + file );
- InputStream testActivityFileStream = RiakHttpPersistIT.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
- testPersistWriter.write( datum );
- LOGGER.info("Wrote: " + activity.getVerb() );
- count++;
- }
-
- testPersistWriter.cleanUp();
-
- LOGGER.info("Total Written: {}", count );
- Assert.assertEquals(count, 89);
-
- RiakHttpPersistReader testPersistReader = new RiakHttpPersistReader(testConfiguration);
- testPersistReader.prepare(testConfiguration);
-
- StreamsResultSet readerResultSet = testPersistReader.readAll();
- LOGGER.info("Total Read: {}", readerResultSet.size() );
- Assert.assertEquals(readerResultSet.size(), 89);
-
- }
-
-}
diff --git a/streams-contrib/streams-persist-riak/src/test/resources/RiakBinaryPersistIT.conf b/streams-contrib/streams-persist-riak/src/test/resources/RiakBinaryPersistIT.conf
deleted file mode 100644
index 2957ab6..0000000
--- a/streams-contrib/streams-persist-riak/src/test/resources/RiakBinaryPersistIT.conf
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-riak {
- scheme = "tcp"
- hosts += ${riak.tcp.host}
- port = ${riak.tcp.port}
- defaultBucket = "binary"
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-persist-riak/src/test/resources/RiakHttpPersistIT.conf b/streams-contrib/streams-persist-riak/src/test/resources/RiakHttpPersistIT.conf
deleted file mode 100644
index 83f097f..0000000
--- a/streams-contrib/streams-persist-riak/src/test/resources/RiakHttpPersistIT.conf
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-riak {
- scheme = "http"
- hosts += ${riak.http.host}
- port = ${riak.http.port}
- defaultBucket = "http"
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-processor-jackson/README.md b/streams-contrib/streams-processor-jackson/README.md
deleted file mode 100644
index 5808a9a..0000000
--- a/streams-contrib/streams-processor-jackson/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-processor-jackson
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-processor-jackson/pom.xml b/streams-contrib/streams-processor-jackson/pom.xml
deleted file mode 100644
index f4e2fe0..0000000
--- a/streams-contrib/streams-processor-jackson/pom.xml
+++ /dev/null
@@ -1,94 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-processor-jackson</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Jackson Utility Processors</description>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path-assert</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
-
- </plugins>
- </build>
-
-</project>
diff --git a/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java b/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java
deleted file mode 100644
index 21a1d49..0000000
--- a/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.jackson;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This processor walks an input objectnode and corrects any artifacts
- * that may have occured from improper serialization of jackson beans.
- *
- * <p/>
- * The logic is also available for inclusion in other module via static import.
- */
-public class CleanAdditionalPropertiesProcessor implements StreamsProcessor {
-
- public static final String STREAMS_ID = "CleanAdditionalPropertiesProcessor";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(CleanAdditionalPropertiesProcessor.class);
-
- private ObjectMapper mapper;
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum datum) {
- List<StreamsDatum> result = new LinkedList<>();
- ObjectNode activity = this.mapper.convertValue(datum.getDocument(), ObjectNode.class);
- cleanAdditionalProperties(activity);
- datum.setDocument(activity);
- result.add(datum);
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.mapper = StreamsJacksonMapper.getInstance();
- }
-
- @Override
- public void cleanUp() {
-
- }
-
- /**
- * Recursively removes all additionalProperties maps.
- * @param node ObjectNode
- */
- public static void cleanAdditionalProperties(ObjectNode node) {
- if ( node.get("additionalProperties") != null ) {
- ObjectNode additionalProperties = (ObjectNode) node.get("additionalProperties");
- cleanAdditionalProperties(additionalProperties);
- Iterator<Map.Entry<String, JsonNode>> jsonNodeIterator = additionalProperties.fields();
- while ( jsonNodeIterator.hasNext() ) {
- Map.Entry<String, JsonNode> entry = jsonNodeIterator.next();
- node.put(entry.getKey(), entry.getValue());
- }
- }
- }
-}
diff --git a/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/JsonUtil.java b/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/JsonUtil.java
deleted file mode 100644
index 43e5879..0000000
--- a/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/JsonUtil.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * JSON utilities.
- */
-public class JsonUtil {
-
- private JsonUtil() {}
-
- private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- private static JsonFactory factory = mapper.getFactory();
-
- public static JsonNode jsonToJsonNode(String json) {
- JsonNode node;
- try {
- JsonParser jp = factory.createJsonParser(json);
- node = mapper.readTree(jp);
- } catch (IOException e) {
- throw new RuntimeException("IO exception while reading JSON", e);
- }
- return node;
- }
-
- public static String jsonNodeToJson(JsonNode node) {
- try {
- return mapper.writeValueAsString(node);
- } catch (JsonProcessingException e) {
- throw new RuntimeException("IO exception while writing JSON", e);
- }
- }
-
- public static <T> T jsonToObject(String json, Class<T> clazz) {
- try {
- return mapper.readValue(json, clazz);
- } catch (IOException e) {
- throw new RuntimeException("Could not map to object");
- }
- }
-
- public static <T> T jsonNodeToObject(JsonNode node, Class<T> clazz) {
- return mapper.convertValue(node, clazz);
- }
-
- public static <T> JsonNode objectToJsonNode(T obj) {
- return mapper.valueToTree(obj);
- }
-
- public static <T> List<T> jsoNodeToList(JsonNode node, Class<T> clazz) {
- return mapper.convertValue(node, new TypeReference<List<T>>() {});
- }
-
- public static <T> String objectToJson(T object) {
- try {
- return mapper.writeValueAsString(object);
- } catch (IOException e) {
- throw new RuntimeException("Could not map to object");
- }
- }
-
- public static <T> T getObjFromFile(String filePath, Class<T> clazz) {
- return jsonNodeToObject(getFromFile(filePath), clazz);
- }
-
- public static JsonNode getFromFile(String filePath) {
- JsonFactory factory = mapper.getFactory(); // since 2.1 use mapper.getFactory() instead
-
- JsonNode node;
- try {
- InputStream stream = getStreamForLocation(filePath);
- JsonParser jp = factory.createParser(stream);
- node = mapper.readTree(jp);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return node;
- }
-
- private static InputStream getStreamForLocation(String filePath) throws FileNotFoundException {
- InputStream stream;
- if(filePath.startsWith("file:///")) {
- stream = new FileInputStream(filePath.replace("file:///", ""));
- } else if(filePath.startsWith("file:") || filePath.startsWith("/")) {
- stream = new FileInputStream(filePath.replace("file:", ""));
- } else {
- //Assume classpath
- stream = JsonUtil.class.getClassLoader().getResourceAsStream(filePath.replace("classpath:", ""));
- }
-
- return stream;
- }
-
- /**
- * Creates an empty array if missing
- * @param node object to create the array within
- * @param field location to create the array
- * @return the Map representing the extensions property
- */
- public static ArrayNode ensureArray(ObjectNode node, String field) {
- String[] path = field.split(".");
- ObjectNode current = node;
- ArrayNode result;
- for( int i = 0; i < path.length; i++) {
- current = ensureObject((ObjectNode) node.get(path[i]), path[i]);
- }
- if (current.get(field) == null)
- current.put(field, mapper.createArrayNode());
- result = (ArrayNode) node.get(field);
- return result;
- }
-
- /**
- * Creates an empty array if missing
- * @param node objectnode to create the object within
- * @param field location to create the object
- * @return the Map representing the extensions property
- */
- public static ObjectNode ensureObject(ObjectNode node, String field) {
- String[] path = field.split(".");
- ObjectNode current = node;
- ObjectNode result;
- for( int i = 0; i < path.length; i++) {
- if (node.get(field) == null)
- node.put(field, mapper.createObjectNode());
- current = (ObjectNode) node.get(field);
- }
- result = ensureObject((ObjectNode) node.get(path[path.length]), String.join(".", Arrays.copyOfRange(path, 1, path.length)));
- return result;
- }
-
-}
diff --git a/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/TypeConverterProcessor.java b/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/TypeConverterProcessor.java
deleted file mode 100644
index 26c7c4f..0000000
--- a/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/TypeConverterProcessor.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.streams.jackson;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * TypeConverterProcessor changes the JVM type while maintaining
- * the underlying document.
- */
-public class TypeConverterProcessor implements StreamsProcessor {
-
- public static final String STREAMS_ID = "TypeConverterProcessor";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TypeConverterProcessor.class);
-
- private List<String> formats = new ArrayList<>();
-
- private ObjectMapper mapper;
-
- private Class inClass;
- private Class outClass;
-
-
- /**
- * TypeConverterProcessor constructor.
- * @param inClass inClass
- * @param outClass outClass
- * @param mapper mapper
- */
- public TypeConverterProcessor(Class inClass, Class outClass, ObjectMapper mapper) {
- this.inClass = inClass;
- this.outClass = outClass;
- this.mapper = mapper;
- }
-
- /**
- * TypeConverterProcessor constructor.
- * @param inClass inClass
- * @param outClass outClass
- * @param formats formats
- */
- public TypeConverterProcessor(Class inClass, Class outClass, List<String> formats) {
- this.inClass = inClass;
- this.outClass = outClass;
- this.formats = formats;
- }
-
- /**
- * TypeConverterProcessor constructor.
- * @param inClass inClass
- * @param outClass outClass
- */
- public TypeConverterProcessor(Class inClass, Class outClass) {
- this.inClass = inClass;
- this.outClass = outClass;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- List<StreamsDatum> result = new LinkedList<>();
- Object inDoc = entry.getDocument();
- ObjectNode node = null;
- if ( inClass == String.class
- || inDoc instanceof String ) {
- try {
- node = this.mapper.readValue((String)entry.getDocument(), ObjectNode.class);
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- } else {
- node = this.mapper.convertValue(inDoc, ObjectNode.class);
- }
-
- if (node != null) {
- Object outDoc;
- try {
- if ( outClass == String.class ) {
- outDoc = this.mapper.writeValueAsString(node);
- } else {
- outDoc = this.mapper.convertValue(node, outClass);
- }
- StreamsDatum outDatum = new StreamsDatum(outDoc, entry.getId(), entry.getTimestamp(), entry.getSequenceid());
- outDatum.setMetadata(entry.getMetadata());
- result.add(outDatum);
- } catch (Throwable ex) {
- LOGGER.warn(ex.getMessage());
- LOGGER.warn(node.toString());
- }
- }
-
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- if ( formats.size() > 0 ) {
- this.mapper = StreamsJacksonMapper.getInstance(formats);
- } else {
- this.mapper = StreamsJacksonMapper.getInstance();
- }
- }
-
- @Override
- public void cleanUp() {
-
- }
-}
diff --git a/streams-contrib/streams-processor-jackson/src/main/resources/components.dot b/streams-contrib/streams-processor-jackson/src/main/resources/components.dot
deleted file mode 100644
index fbe282b..0000000
--- a/streams-contrib/streams-processor-jackson/src/main/resources/components.dot
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- persisters [label="persisters", shape="circle"];
-
- subgraph cluster_processors {
- label="processors";
- processors_jackson_CleanAdditionalPropertiesProcessor [label="CleanAdditionalPropertiesProcessor"]
- processors_jackson_TypeConverterProcessor [label="TypeConverterProcessor"]
- }
-
- generators -> providers
-
- providers -> processors_jackson_CleanAdditionalPropertiesProcessor [label="StreamsDatum"]
- processors_jackson_CleanAdditionalPropertiesProcessor -> persisters [label="StreamsDatum"]
-
- providers -> processors_jackson_TypeConverterProcessor [label="StreamsDatum"]
- processors_jackson_TypeConverterProcessor -> persisters [label="StreamsDatum"]
-
-}
diff --git a/streams-contrib/streams-processor-jackson/src/site/markdown/index.md b/streams-contrib/streams-processor-jackson/src/site/markdown/index.md
deleted file mode 100644
index cd217c2..0000000
--- a/streams-contrib/streams-processor-jackson/src/site/markdown/index.md
+++ /dev/null
@@ -1,17 +0,0 @@
-streams-processor-jackson
-=====================
-
-Jackson Utility Processors
-
-## Components
-
-
-
-| Class |
-|-------|
-| CleanAdditionalPropertiesProcesor [CleanAdditionalPropertiesProcesor.html](apidocs/org/apache/streams/jackson/CleanAdditionalPropertiesProcesor.html "javadoc")
-| TypeConverterProcessor [TypeConverterProcessor.html](apidocs/org/apache/streams/jackson/TypeConverterProcessor.html "javadoc")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-processor-jackson/src/test/java/org/apache/streams/jackson/test/TypeConverterProcessorTest.java b/streams-contrib/streams-processor-jackson/src/test/java/org/apache/streams/jackson/test/TypeConverterProcessorTest.java
deleted file mode 100644
index cea963a..0000000
--- a/streams-contrib/streams-processor-jackson/src/test/java/org/apache/streams/jackson/test/TypeConverterProcessorTest.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson.test;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.jackson.TypeConverterProcessor;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- *
- */
-public class TypeConverterProcessorTest {
-
- private static final String DATASIFT_JSON = "{\"demographic\":{\"gender\":\"female\"},\"interaction\":{\"schema\":{\"version\":3},\"source\":\"Twitter for Android\",\"author\":{\"username\":\"ViiOLeee\",\"name\":\"Violeta Anguita\",\"id\":70931384,\"avatar\":\"http://pbs.twimg.com/profile_images/378800000851401229/bbf480cde2e9923a1d20acd393da0212_normal.jpeg\",\"link\":\"http://twitter.com/ViiOLeee\",\"language\":\"en\"},\"type\":\"twitter\",\"created_at\":\"Tue, 27 May 2014 22:38:15 + [...]
-
- public static final String DATASIFT_FORMAT = "EEE, dd MMM yyyy HH:mm:ss Z";
-
- @Test
- public void testTypeConverterStringToString() {
- final String ID = "1";
- StreamsProcessor processor = new TypeConverterProcessor(String.class, String.class, Collections.singletonList(DATASIFT_FORMAT));
- processor.prepare(null);
- StreamsDatum datum = new StreamsDatum(DATASIFT_JSON, ID);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof String);
- assertEquals(ID, resultDatum.getId());
- }
-
- @Test
- public void testTypeConverterStringToObjectNode() {
- final String ID = "1";
- StreamsProcessor processor = new TypeConverterProcessor(String.class, ObjectNode.class, Collections.singletonList(DATASIFT_FORMAT));
- processor.prepare(null);
- StreamsDatum datum = new StreamsDatum(DATASIFT_JSON, ID);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof ObjectNode);
- assertEquals(ID, resultDatum.getId());
- }
-
- @Test
- public void testTypeConverterObjectNodeToString() throws IOException {
- final String ID = "1";
- StreamsProcessor processor = new TypeConverterProcessor(ObjectNode.class, String.class, Collections.singletonList(DATASIFT_FORMAT));
- processor.prepare(null);
- ObjectMapper mapper = StreamsJacksonMapper.getInstance(Collections.singletonList(DATASIFT_FORMAT));
- ObjectNode node = mapper.readValue(DATASIFT_JSON, ObjectNode.class);
- StreamsDatum datum = new StreamsDatum(node, ID);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertNotNull(resultDatum.getDocument());
- assertTrue(resultDatum.getDocument() instanceof String);
- assertEquals(ID, resultDatum.getId());
- }
-}
diff --git a/streams-contrib/streams-processor-json/README.md b/streams-contrib/streams-processor-json/README.md
deleted file mode 100644
index 1ce236d..0000000
--- a/streams-contrib/streams-processor-json/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-processor-json
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-processor-json/pom.xml b/streams-contrib/streams-processor-json/pom.xml
deleted file mode 100644
index d912454..0000000
--- a/streams-contrib/streams-processor-json/pom.xml
+++ /dev/null
@@ -1,104 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-processor-json</artifactId>
- <name>${project.artifactId}</name>
-
- <description>JsonPath Utility Processors</description>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path-assert</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes combine.children="append">
- <exclude>src/test/resources/books.json</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-
-</project>
diff --git a/streams-contrib/streams-processor-json/src/main/java/org/apache/streams/json/JsonPathExtractor.java b/streams-contrib/streams-processor-json/src/main/java/org/apache/streams/json/JsonPathExtractor.java
deleted file mode 100644
index 3c680bd..0000000
--- a/streams-contrib/streams-processor-json/src/main/java/org/apache/streams/json/JsonPathExtractor.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.json;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.jayway.jsonpath.JsonPath;
-import net.minidev.json.JSONArray;
-import net.minidev.json.JSONObject;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-
-/**
- * Provides a base implementation for extracting json fields and
- * objects from datums using JsonPath syntax.
- */
-public class JsonPathExtractor implements StreamsProcessor {
-
- private static final String STREAMS_ID = "JsonPathExtractor";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(JsonPathExtractor.class);
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private String pathExpression;
- private JsonPath jsonPath;
-
- public JsonPathExtractor() {
- LOGGER.info("creating JsonPathExtractor");
- }
-
- public JsonPathExtractor(String pathExpression) {
- this.pathExpression = pathExpression;
- LOGGER.info("creating JsonPathExtractor for " + this.pathExpression);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = new ArrayList<>();
-
- String json = null;
-
- LOGGER.debug("{} processing {}", STREAMS_ID);
-
- if ( entry.getDocument() instanceof ObjectNode ) {
- ObjectNode node = (ObjectNode) entry.getDocument();
- try {
- json = mapper.writeValueAsString(node);
- } catch (JsonProcessingException ex) {
- LOGGER.warn(ex.getMessage());
- }
- } else if ( entry.getDocument() instanceof String ) {
- json = (String) entry.getDocument();
- }
-
- if ( StringUtils.isNotEmpty(json)) {
-
- try {
- Object readResult = jsonPath.read(json);
-
- if (readResult instanceof String) {
- String match = (String) readResult;
- LOGGER.info("Matched String: " + match);
- StreamsDatum matchDatum = new StreamsDatum(match);
- result.add(matchDatum);
- } else if (readResult instanceof JSONObject) {
- JSONObject match = (JSONObject) readResult;
- LOGGER.info("Matched Object: " + match);
- ObjectNode objectNode = mapper.readValue(mapper.writeValueAsString(match), ObjectNode.class);
- StreamsDatum matchDatum = new StreamsDatum(objectNode);
- result.add(matchDatum);
- } else if (readResult instanceof JSONArray) {
- LOGGER.info("Matched Array:");
- JSONArray array = (JSONArray) readResult;
- for (Object item : array) {
- if (item instanceof String) {
- LOGGER.info("String Item:" + item);
- String match = (String) item;
- StreamsDatum matchDatum = new StreamsDatum(match);
- result.add(matchDatum);
- } else if (item instanceof JSONObject) {
- LOGGER.info("Object Item:" + item);
- JSONObject match = (JSONObject) item;
- ObjectNode objectNode = mapper.readValue(mapper.writeValueAsString(match), ObjectNode.class);
- StreamsDatum matchDatum = new StreamsDatum(objectNode);
- result.add(matchDatum);
- } else if (item instanceof LinkedHashMap) {
- LOGGER.info("Object Item:" + item);
- ObjectNode objectNode = mapper.readValue(mapper.writeValueAsString(item), ObjectNode.class);
- StreamsDatum matchDatum = new StreamsDatum(objectNode);
- result.add(matchDatum);
- } else {
- LOGGER.info("Other Item:" + item.toString());
- }
- }
- } else {
- LOGGER.info("Other Match:" + readResult.toString());
- }
-
- } catch ( Exception ex ) {
- LOGGER.warn(ex.getMessage());
- }
-
- } else {
- LOGGER.warn("result empty");
- }
-
- return result;
-
- }
-
- @Override
- public void prepare(Object configurationObject) {
- if ( configurationObject instanceof String ) {
- jsonPath = JsonPath.compile((String) (configurationObject));
- } else if ( configurationObject instanceof String[] ) {
- jsonPath = JsonPath.compile(((String[]) (configurationObject))[0]);
- }
- }
-
- @Override
- public void cleanUp() {
- LOGGER.info("shutting down JsonPathExtractor for " + this.pathExpression);
- }
-}
diff --git a/streams-contrib/streams-processor-json/src/main/java/org/apache/streams/json/JsonPathFilter.java b/streams-contrib/streams-processor-json/src/main/java/org/apache/streams/json/JsonPathFilter.java
deleted file mode 100644
index 1e3a5bf..0000000
--- a/streams-contrib/streams-processor-json/src/main/java/org/apache/streams/json/JsonPathFilter.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.json;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.jayway.jsonpath.JsonPath;
-import net.minidev.json.JSONArray;
-import net.minidev.json.JSONObject;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * Provides a base implementation for filtering datums which
- * do not contain specific fields using JsonPath syntax.
- */
-public class JsonPathFilter implements StreamsProcessor {
-
- private static final String STREAMS_ID = "JsonPathFilter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(JsonPathFilter.class);
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private String pathExpression;
- private JsonPath jsonPath;
- private String destNodeName;
-
- public JsonPathFilter() {
- LOGGER.info("creating JsonPathFilter");
- }
-
- public JsonPathFilter(String pathExpression) {
- this.pathExpression = pathExpression;
- LOGGER.info("creating JsonPathFilter for " + this.pathExpression);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = new ArrayList<>();
-
- String json = null;
-
- ObjectNode document = null;
-
- LOGGER.debug("{} processing {}", STREAMS_ID);
-
- if ( entry.getDocument() instanceof ObjectNode ) {
- document = (ObjectNode) entry.getDocument();
- try {
- json = mapper.writeValueAsString(document);
- } catch (JsonProcessingException ex) {
- ex.printStackTrace();
- }
- } else if ( entry.getDocument() instanceof String ) {
- json = (String) entry.getDocument();
- try {
- document = mapper.readValue(json, ObjectNode.class);
- } catch (IOException ex) {
- ex.printStackTrace();
- return null;
- }
- }
-
- Objects.requireNonNull(document);
-
- if ( StringUtils.isNotEmpty(json)) {
-
- Object srcResult = null;
- try {
- srcResult = jsonPath.read(json);
- } catch ( Exception ex ) {
- ex.printStackTrace();
- LOGGER.warn(ex.getMessage());
- }
-
- Objects.requireNonNull(srcResult);
-
- String[] path = StringUtils.split(pathExpression, '.');
- ObjectNode node = document;
- for (int i = 1; i < path.length - 1; i++) {
- node = (ObjectNode) document.get(path[i]);
- }
-
- Objects.requireNonNull(node);
-
- if ( srcResult instanceof JSONArray ) {
- try {
- ArrayNode jsonNode = mapper.convertValue(srcResult, ArrayNode.class);
- if ( jsonNode.size() == 1 ) {
- JsonNode item = jsonNode.get(0);
- node.set(destNodeName, item);
- } else {
- node.set(destNodeName, jsonNode);
- }
- } catch (Exception ex) {
- LOGGER.warn(ex.getMessage());
- }
- } else if ( srcResult instanceof JSONObject ) {
- try {
- ObjectNode jsonNode = mapper.convertValue(srcResult, ObjectNode.class);
- node.set(destNodeName, jsonNode);
- } catch (Exception ex) {
- LOGGER.warn(ex.getMessage());
- }
- } else if ( srcResult instanceof String ) {
- try {
- node.put(destNodeName, (String) srcResult);
- } catch (Exception ex) {
- LOGGER.warn(ex.getMessage());
- }
- }
-
- }
-
- result.add(new StreamsDatum(document));
-
- return result;
-
- }
-
- @Override
- public void prepare(Object configurationObject) {
- if ( configurationObject instanceof Map) {
- Map<String,String> params = ( Map<String,String>) configurationObject;
- pathExpression = params.get("pathExpression");
- jsonPath = JsonPath.compile(pathExpression);
- destNodeName = pathExpression.substring(pathExpression.lastIndexOf(".") + 1);
- }
- }
-
- @Override
- public void cleanUp() {
- LOGGER.info("shutting down JsonPathFilter for " + this.pathExpression);
- }
-}
diff --git a/streams-contrib/streams-processor-json/src/main/resources/components.dot b/streams-contrib/streams-processor-json/src/main/resources/components.dot
deleted file mode 100644
index 1fdf562..0000000
--- a/streams-contrib/streams-processor-json/src/main/resources/components.dot
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- persisters [label="persisters", shape="circle"];
-
- subgraph cluster_processors {
- label="processors";
- processors_json_JsonPathExtractor [label="JsonPathExtractor"]
- processors_json_JsonPathFilter [label="JsonPathFilter"]
- }
-
- generators -> providers
-
- providers -> processors_json_JsonPathExtractor [label="StreamsDatum[String]\nStreamsDatum[ObjectNode]"]
- processors_json_JsonPathExtractor -> persisters [label="StreamsDatum[String]\nStreamsDatum[ObjectNode]"]
-
- providers -> processors_json_JsonPathFilter [label="StreamsDatum[ObjectNode]"]
- processors_json_JsonPathFilter -> persisters [label="StreamsDatum[ObjectNode]"]
-
-}
diff --git a/streams-contrib/streams-processor-json/src/site/markdown/index.md b/streams-contrib/streams-processor-json/src/site/markdown/index.md
deleted file mode 100644
index 9b01208..0000000
--- a/streams-contrib/streams-processor-json/src/site/markdown/index.md
+++ /dev/null
@@ -1,17 +0,0 @@
-streams-processor-json
-=====================
-
-JsonPath Utility Processors
-
-## Components
-
-
-
-| Class |
-|-------|
-| JsonPathExtractor [JsonPathExtractor.html](apidocs/org/apache/streams/jackson/JsonPathExtractor.html "javadoc")
-| JsonPathFilter [JsonPathFilter.html](apidocs/org/apache/streams/jackson/JsonPathFilter.html "javadoc")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-processor-json/src/test/java/org/apache/streams/json/test/JsonPathExtractorTest.java b/streams-contrib/streams-processor-json/src/test/java/org/apache/streams/json/test/JsonPathExtractorTest.java
deleted file mode 100644
index 50ea9ba..0000000
--- a/streams-contrib/streams-processor-json/src/test/java/org/apache/streams/json/test/JsonPathExtractorTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.json.test;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.json.JsonPathExtractor;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.io.FileUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for extracting json fields and
- * objects from datums using JsonPath syntax.
- */
-public class JsonPathExtractorTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(JsonPathExtractorTest.class);
-
- private String testJson;
-
- @Before
- public void initialize() {
- try {
- testJson = FileUtils.readFileToString(new File("src/test/resources/books.json"));
- } catch (IOException ex) {
- ex.printStackTrace();
- Assert.fail();
- }
- }
-
- @Test
- public void test1()
- {
- JsonPathExtractor extractor = new JsonPathExtractor();
- extractor.prepare("$.store.book[*].author");
- List<StreamsDatum> result = extractor.process(new StreamsDatum(testJson));
- assertThat(result.size(), is(2));
- assertTrue(result.get(0).getDocument() instanceof String);
- assertTrue(result.get(1).getDocument() instanceof String);
- }
-
- @Test
- public void test2()
- {
- JsonPathExtractor extractor = new JsonPathExtractor();
- extractor.prepare("$.store.book[?(@.category == 'reference')]");
- List<StreamsDatum> result = extractor.process(new StreamsDatum(testJson));
- assertThat(result.size(), is(1));
- assertTrue(result.get(0).getDocument() instanceof ObjectNode);
- }
-
- @Test
- public void test3()
- {
- JsonPathExtractor extractor = new JsonPathExtractor();
- extractor.prepare("$.store.book[?(@.price > 10)]");
- List<StreamsDatum> result = extractor.process(new StreamsDatum(testJson));
- assertThat(result.size(), is(1));
- assertTrue(result.get(0).getDocument() instanceof ObjectNode);
- }
-
- @Test
- public void test4()
- {
- JsonPathExtractor extractor = new JsonPathExtractor();
- extractor.prepare("$.store.book[?(@.isbn)]");
- List<StreamsDatum> result = extractor.process(new StreamsDatum(testJson));
- assertThat(result.size(), is(1));
- assertTrue(result.get(0).getDocument() instanceof ObjectNode);
- }
-
-}
diff --git a/streams-contrib/streams-processor-json/src/test/resources/books.json b/streams-contrib/streams-processor-json/src/test/resources/books.json
deleted file mode 100644
index 30cf954..0000000
--- a/streams-contrib/streams-processor-json/src/test/resources/books.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "store": {
- "book": [
- { "category": "reference",
- "author": "Nigel Rees",
- "title": "Sayings of the Century",
- "price": 8.95
- },
- { "category": "fiction",
- "author": "Evelyn Waugh",
- "title": "Sword of Honour",
- "price": 12.99,
- "isbn": "0-553-21311-3"
- }
- ],
- "bicycle": {
- "color": "red",
- "price": 19.95
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-processor-peoplepattern/README.md b/streams-contrib/streams-processor-peoplepattern/README.md
deleted file mode 100644
index a29a734..0000000
--- a/streams-contrib/streams-processor-peoplepattern/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-processor-peoplepattern
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-processor-peoplepattern/pom.xml b/streams-contrib/streams-processor-peoplepattern/pom.xml
deleted file mode 100644
index 81911b2..0000000
--- a/streams-contrib/streams-processor-peoplepattern/pom.xml
+++ /dev/null
@@ -1,140 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-processor-peoplepattern</artifactId>
- <name>${project.artifactId}</name>
-
- <description>PeoplePattern API Processors</description>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-http</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jsonschema2pojo/**/*.java</source>
- </sources>
- </configuration>
- </execution>
- <execution>
- <id>add-source-jaxb2</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/jaxb2</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <addCompileSourceRoot>true</addCompileSourceRoot>
- <generateBuilders>true</generateBuilders>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema/org/apache/streams/peoplepattern</sourcePath>
- </sourcePaths>
- <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
- <targetPackage>org.apache.streams.peoplepattern</targetPackage>
- <useLongIntegers>true</useLongIntegers>
- <useJodaDates>true</useJodaDates>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
-</project>
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
deleted file mode 100644
index 04a680d..0000000
--- a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/AccountTypeProcessor.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.peoplepattern;
-
-import org.apache.streams.components.http.HttpProcessorConfiguration;
-import org.apache.streams.components.http.processor.SimpleHTTPGetProcessor;
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Enrich actor with account type.
- */
-public class AccountTypeProcessor extends SimpleHTTPGetProcessor {
-
- private static final String STREAMS_ID = "AccountTypeProcessor";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(AccountTypeProcessor.class);
-
- /**
- * AccountTypeProcessor constructor - resolves HttpProcessorConfiguration from JVM 'peoplepattern'.
- */
- public AccountTypeProcessor() {
- this(new ComponentConfigurator<>(HttpProcessorConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("peoplepattern")));
- }
-
- /**
- * AccountTypeProcessor constructor - uses supplied HttpProcessorConfiguration.
- * @param peoplePatternConfiguration peoplePatternConfiguration
- */
- public AccountTypeProcessor(HttpProcessorConfiguration peoplePatternConfiguration) {
- super(peoplePatternConfiguration);
- LOGGER.info("creating AccountTypeProcessor");
- configuration.setProtocol("https");
- configuration.setHostname("api.peoplepattern.com");
- configuration.setResourcePath("/v0.2/account_type/");
- configuration.setEntity(HttpProcessorConfiguration.Entity.ACTOR);
- configuration.setExtension("account_type");
- }
-
- /**
- Override this to add parameters to the request.
- */
- @Override
- protected Map<String, String> prepareParams(StreamsDatum entry) {
- Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
- ActivityObject actor = mapper.convertValue(activity.getActor(), ActivityObject.class);
- String username = (String) ExtensionUtil.getInstance().getExtension(actor, "screenName");
- Map<String, String> params = new HashMap<>();
- params.put("id", actor.getId());
- params.put("name", actor.getDisplayName());
- params.put("username", username);
- params.put("description", actor.getSummary());
- return params;
- }
-}
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java b/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
deleted file mode 100644
index e615026..0000000
--- a/streams-contrib/streams-processor-peoplepattern/src/main/java/org/apache/streams/peoplepattern/DemographicsProcessor.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.peoplepattern;
-
-import org.apache.streams.components.http.HttpProcessorConfiguration;
-import org.apache.streams.components.http.processor.SimpleHTTPGetProcessor;
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Enrich actor with demographics.
- */
-public class DemographicsProcessor extends SimpleHTTPGetProcessor {
-
- public static final String STREAMS_ID = "DemographicsProcessor";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(DemographicsProcessor.class);
-
- /**
- * DemographicsProcessor constructor - resolves HttpProcessorConfiguration from JVM 'peoplepattern'.
- */
- public DemographicsProcessor() {
- this(new ComponentConfigurator<>(HttpProcessorConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("peoplepattern")));
- }
-
- /**
- * AccountTypeProcessor constructor - uses supplied HttpProcessorConfiguration.
- * @param peoplePatternConfiguration peoplePatternConfiguration
- */
- public DemographicsProcessor(HttpProcessorConfiguration peoplePatternConfiguration) {
- super(peoplePatternConfiguration);
- LOGGER.info("creating DemographicsProcessor");
- configuration.setProtocol("https");
- configuration.setHostname("api.peoplepattern.com");
- configuration.setResourcePath("/v0.2/demographics/");
- configuration.setEntity(HttpProcessorConfiguration.Entity.ACTOR);
- configuration.setExtension("demographics");
- }
-
- /**
- Override this to add parameters to the request.
- */
- @Override
- protected Map<String, String> prepareParams(StreamsDatum entry) {
- Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
- ActivityObject actor = mapper.convertValue(activity.getActor(), ActivityObject.class);
- String username = (String) ExtensionUtil.getInstance().getExtension(actor, "screenName");
- Map<String, String> params = new HashMap<>();
- params.put("id", actor.getId());
- params.put("name", actor.getDisplayName());
- params.put("username", username);
- params.put("description", actor.getSummary());
- return params;
- }
-
-}
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/AccountType.json b/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/AccountType.json
deleted file mode 100644
index 8c8d7e9..0000000
--- a/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/AccountType.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.peoplepattern.AccountType",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "prediction" : {
- "type" : "string",
- "enum" : [
- "person",
- "organization",
- "entertainment",
- "adult",
- "spam",
- "no-prediction"
- ],
- "default": "no-prediction"
- },
- "score": {
- "type": "number"
- },
- "id": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/Demographics.json b/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/Demographics.json
deleted file mode 100644
index bf8272f..0000000
--- a/streams-contrib/streams-processor-peoplepattern/src/main/jsonschema/org/apache/streams/peoplepattern/Demographics.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.peoplepattern.Demographics",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "age": {
- "type": "object",
- "properties": {
- "prediction": {
- "type": "integer",
- "default": 1990
- },
- "score": {
- "type": "number"
- }
-
- }
- },
- "gender" : {
- "type": "object",
- "properties": {
- "prediction": {
- "type": "string",
- "enum": [
- "male",
- "female",
- "no-prediction"
- ],
- "default": "no-prediction"
- },
- "score": {
- "type": "number"
- }
- }
- },
- "race" : {
- "type": "object",
- "properties": {
- "prediction": {
- "type": "string",
- "enum": [
- "black",
- "east-asian",
- "hispanic",
- "middle-eastern",
- "south-asian",
- "white",
- "no-prediction"
- ],
- "default": "no-prediction"
- },
- "score": {
- "type": "number"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-processor-peoplepattern/src/main/resources/templates/peoplepatternactor.json b/streams-contrib/streams-processor-peoplepattern/src/main/resources/templates/peoplepatternactor.json
deleted file mode 100644
index 21285e4..0000000
--- a/streams-contrib/streams-processor-peoplepattern/src/main/resources/templates/peoplepatternactor.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "order": 20,
- "template": "*activity*",
- "settings": {},
- "mappings": {
- "activity": {
- "properties": {
- "actor": {
- "properties": {
- "extensions": {
- "properties": {
- "account_type": {
- "type": "nested"
- },
- "demographics": {
- "type": "nested"
- }
- }
- }
- }
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-processor-peoplepattern/src/site/markdown/index.md b/streams-contrib/streams-processor-peoplepattern/src/site/markdown/index.md
deleted file mode 100644
index 899ce9b..0000000
--- a/streams-contrib/streams-processor-peoplepattern/src/site/markdown/index.md
+++ /dev/null
@@ -1,15 +0,0 @@
-streams-processor-peoplepattern
-=====================
-
-PeoplePattern API Processors
-
-Example configuration:
-
- "peoplepattern": {
- "username": "",
- "password": ""
- }
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-processor-regex/README.md b/streams-contrib/streams-processor-regex/README.md
deleted file mode 100644
index 1dd60e5..0000000
--- a/streams-contrib/streams-processor-regex/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-processor-regex
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-processor-regex/pom.xml b/streams-contrib/streams-processor-regex/pom.xml
deleted file mode 100644
index 1b25edd..0000000
--- a/streams-contrib/streams-processor-regex/pom.xml
+++ /dev/null
@@ -1,57 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-processor-regex</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Regex Utility Processors</description>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/AbstractRegexExtensionExtractor.java b/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/AbstractRegexExtensionExtractor.java
deleted file mode 100644
index 305ddfd..0000000
--- a/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/AbstractRegexExtensionExtractor.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.regex;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Sets;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Provides a base implementation for extracting entities from text using regular expressions and then
- * modifying the appropriate {@link org.apache.streams.pojo.json.Activity} extensions object.
- */
-public abstract class AbstractRegexExtensionExtractor<T> implements StreamsProcessor {
-
- private final String patternConfigKey;
- private final String extensionKey;
- private final String defaultPattern;
-
- private static final Logger LOGGER = LoggerFactory.getLogger(AbstractRegexExtensionExtractor.class);
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private String pattern;
-
- protected AbstractRegexExtensionExtractor(String patternConfigKey, String extensionKey, String defaultPattern) {
- this.patternConfigKey = patternConfigKey;
- this.extensionKey = extensionKey;
- this.defaultPattern = defaultPattern;
- }
-
- public String getPattern() {
- return pattern;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- Activity activity;
- if (entry.getDocument() instanceof Activity) {
- activity = (Activity) entry.getDocument();
- } else if (entry.getDocument() instanceof ObjectNode) {
- activity = mapper.convertValue(entry.getDocument(), Activity.class);
- } else {
- return new ArrayList<>();
- }
- if (StringUtils.isBlank(pattern)) {
- prepare(null);
- }
- Map<String, List<Integer>> matches = RegexUtils.extractMatches(pattern, activity.getContent());
- Collection<T> entities = ensureTargetObject(activity);
- for (String key : matches.keySet()) {
- entities.add(prepareObject(key));
- }
-
- Set<T> set = new HashSet<>();
- set.addAll(entities);
- entities.clear();
- entities.addAll(set);
-
- entry.setDocument(activity);
- return Collections.singletonList(entry);
- }
-
- @Override
- public void prepare(Object configurationObject) {
- if (configurationObject instanceof Map) {
- if (((Map) configurationObject).containsKey(patternConfigKey)) {
- pattern = (String) ((Map) configurationObject).get(patternConfigKey);
- }
- } else if (configurationObject instanceof String) {
- pattern = (String) configurationObject;
- } else {
- pattern = defaultPattern;
- }
- }
-
- @Override
- public void cleanUp() {
- //NOP
- }
-
- /**
- * Configures the value to be persisted to the extensions object.
- * @param extracted the value extracted by the regex
- * @return an object representing the appropriate extension
- */
- protected abstract T prepareObject(String extracted);
-
- @SuppressWarnings("unchecked")
- protected Collection<T> ensureTargetObject(Activity activity) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- Set<T> hashtags;
-
- if (extensions.containsKey(extensionKey) && extensions.get(extensionKey) != null) {
- hashtags = Sets.newHashSet((Iterable<T>) extensions.get(extensionKey));
- } else {
- hashtags = new HashSet<>();
- }
-
- extensions.put(extensionKey, hashtags);
-
- return hashtags;
- }
-}
diff --git a/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/RegexHashtagExtractor.java b/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/RegexHashtagExtractor.java
deleted file mode 100644
index 326d5fa..0000000
--- a/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/RegexHashtagExtractor.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.regex;
-
-import org.apache.streams.core.StreamsProcessor;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Processes the content of an {@link org.apache.streams.pojo.json.Activity} object to extract the Hashtags and add
- * them to the appropriate extensions object
- */
-public class RegexHashtagExtractor extends AbstractRegexExtensionExtractor<String> implements StreamsProcessor {
-
- private static final String STREAMS_ID = "RegexHashtagExtractor";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RegexHashtagExtractor.class);
-
- public static final String DEFAULT_PATTERN = "#\\w+";
- public static final String PATTERN_CONFIG_KEY = "HashtagPattern";
- public static final String EXTENSION_KEY = "hashtags";
-
- public RegexHashtagExtractor() {
- super(PATTERN_CONFIG_KEY, EXTENSION_KEY, DEFAULT_PATTERN);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- protected String prepareObject(String extracted) {
- return extracted.substring(1);
- }
-}
diff --git a/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/RegexMentionsExtractor.java b/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/RegexMentionsExtractor.java
deleted file mode 100644
index 093a88d..0000000
--- a/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/RegexMentionsExtractor.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.regex;
-
-import org.apache.streams.core.StreamsProcessor;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Processes the content of an {@link org.apache.streams.pojo.json.Activity} object to extract the @user mentions and add
- * them to the appropriate extensions object
- */
-public class RegexMentionsExtractor extends AbstractRegexExtensionExtractor<Map<String, Object>> implements StreamsProcessor {
-
- private static final String STREAMS_ID = "RegexMentionsExtractor";
-
- public static final String DEFAULT_PATTERN = "@\\w+";
- public static final String PATTERN_CONFIG_KEY = "MentionPattern";
- public static final String EXTENSION_KEY = "user_mentions";
- public static final String DISPLAY_KEY = "displayName";
-
- public RegexMentionsExtractor() {
- super(PATTERN_CONFIG_KEY, EXTENSION_KEY, DEFAULT_PATTERN);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- protected Map<String, Object> prepareObject(String extracted) {
- Map<String, Object> mention = new HashMap<>();
- mention.put(DISPLAY_KEY, extracted.substring(1));
- return mention;
- }
-
-}
diff --git a/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/RegexUrlExtractor.java b/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/RegexUrlExtractor.java
deleted file mode 100644
index ea8474d..0000000
--- a/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/RegexUrlExtractor.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.regex;
-
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.pojo.json.Activity;
-
-import java.util.Collection;
-
-/**
- * Processes the content of an {@link org.apache.streams.pojo.json.Activity} object to extract the URLs and add
- * them to the appropriate extensions object
- */
-public class RegexUrlExtractor extends AbstractRegexExtensionExtractor<String> implements StreamsProcessor {
-
- private static final String STREAMS_ID = "RegexUrlExtractor";
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- //Temporarily copied from streams-processor-urls so as not to force a dependency on that provider. This should
- //be moved to a common utility package
- public static final String DEFAULT_PATTERN =
- "(?:(?:https?|ftp)://)"
- + "(?:\\S+(?::\\S*)?@)?"
- + "(?:"
- + "(?!(?:10|127)(?:\\.\\d{1,3}){3})"
- + "(?!(?:169\\.254|192\\.168)(?:\\.\\d{1,3}){2})"
- + "(?!172\\.(?:1[6-9]|2\\d|3[0-1])(?:\\.\\d{1,3}){2})"
- + "(?:[1-9]\\d?|1\\d\\d|2[01]\\d|22[0-3])"
- + "(?:\\.(?:1?\\d{1,2}|2[0-4]\\d|25[0-5])){2}"
- + "(?:\\.(?:[1-9]\\d?|1\\d\\d|2[0-4]\\d|25[0-4]))"
- + "|"
- + "(?:(?:[a-z\\u00a1-\\uffff0-9]+-?)*[a-z\\u00a1-\\uffff0-9]+)"
- + "(?:\\.(?:[a-z\\u00a1-\\uffff0-9]+-?)*[a-z\\u00a1-\\uffff0-9]+)*"
- + "(?:\\.(?:[a-z\\u00a1-\\uffff]{2,}))"
- + ")"
- + "(?::\\d{2,5})?"
- + "(?:/[^\\s]*)?";
-
- public static final String PATTERN_CONFIG_KEY = "URLPattern";
-
- public RegexUrlExtractor() {
- super(PATTERN_CONFIG_KEY, null, DEFAULT_PATTERN);
- }
-
- @Override
- protected String prepareObject(String extracted) {
- return extracted;
- }
-
- @Override
- protected Collection<String> ensureTargetObject(Activity activity) {
- return activity.getLinks();
- }
-}
diff --git a/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/RegexUtils.java b/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/RegexUtils.java
deleted file mode 100644
index d3d5c89..0000000
--- a/streams-contrib/streams-processor-regex/src/main/java/org/apache/streams/regex/RegexUtils.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.regex;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Provides utilities for extracting matches from content.
- */
-public class RegexUtils {
-
- private static final Map<String, Pattern> patternCache = new ConcurrentHashMap<>();
- private static final Logger LOGGER = LoggerFactory.getLogger(RegexUtils.class);
-
- private RegexUtils() {}
-
- /**
- * Extracts matches of the given pattern in the content and returns them as a list.
- * @param pattern the pattern for the substring to match. For example, [0-9]* matches 911 in Emergency number is 911.
- * @param content the complete content to find matches in.
- * @return a non-null list of matches.
- */
- public static Map<String, List<Integer>> extractMatches(String pattern, String content) {
- return getMatches(pattern, content, -1);
- }
-
- /**
- * Extracts matches of the given pattern that are bounded by separation characters and returns them as a list.
- * @param pattern the pattern for the substring to match. For example, [0-9]* matches 911 in Emergency number is 911.
- * @param content the complete content to find matches in.
- * @return a non-null list of matches.
- */
- public static Map<String, List<Integer>> extractWordMatches(String pattern, String content) {
- pattern = "(^|\\s)(" + pattern + ")([\\s!\\.;,?]|$)";
- return getMatches(pattern, content, 2);
- }
-
- protected static Map<String, List<Integer>> getMatches(String pattern, String content, int capture) {
- try {
- Map<String, List<Integer>> matches = new HashMap<>();
- if (content == null) {
- return matches;
- }
-
- Matcher matcher = getPattern(pattern).matcher(content);
- while (matcher.find()) {
- String group = capture > 0 ? matcher.group(capture) : matcher.group();
- if (group != null && !group.equals("")) {
- List<Integer> indices;
- if (matches.containsKey(group)) {
- indices = matches.get(group);
- } else {
- indices = new ArrayList<>();
- matches.put(group, indices);
- }
- indices.add(matcher.start());
- }
- }
- return matches;
- } catch (Throwable ex) {
- LOGGER.error("Throwable process {}", ex);
- ex.printStackTrace();
- throw new RuntimeException(ex);
- }
- }
-
- private static Pattern getPattern(String patternString) {
- Pattern pattern;
- if (patternCache.containsKey(patternString)) {
- pattern = patternCache.get(patternString);
- } else {
- pattern = Pattern.compile(patternString);
- patternCache.put(patternString, pattern);
- }
- return pattern;
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-processor-regex/src/main/resources/components.dot b/streams-contrib/streams-processor-regex/src/main/resources/components.dot
deleted file mode 100644
index c4b6214..0000000
--- a/streams-contrib/streams-processor-regex/src/main/resources/components.dot
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- persisters [label="persisters", shape="cylinder"];
-
- subgraph cluster_processors {
- label="processors";
- processors_regex_AbstractRegexExtensionExtractor [label="AbstractRegexExtensionExtractor"]
- processors_regex_RegexHashtagExtractor [label="RegexHashtagExtractor"]
- processors_regex_RegexMentionsExtractor [label="RegexMentionsExtractor"]
- processors_regex_RegexUrlExtractor [label="RegexUrlExtractor"]
- }
-
- generators -> providers
-
- providers -> processors_regex_AbstractRegexExtensionExtractor [label="StreamsDatum[Activity]\nStreamsDatum[ObjectNode]"]
- processors_regex_AbstractRegexExtensionExtractor -> persisters [label="StreamsDatum[Activity]"]
-
- providers -> processors_regex_RegexHashtagExtractor [label="StreamsDatum[Activity]\nStreamsDatum[ObjectNode]"]
- processors_regex_RegexHashtagExtractor -> persisters [label="StreamsDatum[Activity]"]
-
- providers -> processors_regex_RegexMentionsExtractor [label="StreamsDatum[Activity]\nStreamsDatum[ObjectNode]"]
- processors_regex_RegexMentionsExtractor -> persisters [label="StreamsDatum[Activity]"]
-
- providers -> processors_regex_RegexUrlExtractor [label="StreamsDatum[Activity]\nStreamsDatum[ObjectNode]"]
- processors_regex_RegexUrlExtractor -> persisters [label="StreamsDatum[Activity]"]
-
-}
diff --git a/streams-contrib/streams-processor-regex/src/site/markdown/index.md b/streams-contrib/streams-processor-regex/src/site/markdown/index.md
deleted file mode 100644
index eef35a7..0000000
--- a/streams-contrib/streams-processor-regex/src/site/markdown/index.md
+++ /dev/null
@@ -1,19 +0,0 @@
-streams-processor-regex
-=====================
-
-Regex Utility Processors
-
-## Components
-
-
-
-| Class |
-|-------|
-| AbstractRegexExtensionExtractor [AbstractRegexExtensionExtractor.html](apidocs/org/apache/streams/regex/AbstractRegexExtensionExtractor.html "javadoc")
-| RegexHashtagExtractor [RegexHashtagExtractor.html](apidocs/org/apache/streams/regex/RegexHashtagExtractor.html "javadoc")
-| RegexMentionsExtractor [RegexMentionsExtractor.html](apidocs/org/apache/streams/regex/RegexMentionsExtractor.html "javadoc")
-| RegexUrlExtractor [RegexUrlExtractor.html](apidocs/org/apache/streams/regex/RegexUrlExtractor.html "javadoc")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-processor-regex/src/test/java/org/apache/streams/regex/RegexHashtagExtractorTest.java b/streams-contrib/streams-processor-regex/src/test/java/org/apache/streams/regex/RegexHashtagExtractorTest.java
deleted file mode 100644
index b17df37..0000000
--- a/streams-contrib/streams-processor-regex/src/test/java/org/apache/streams/regex/RegexHashtagExtractorTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.regex;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-
-@RunWith(Parameterized.class)
-public class RegexHashtagExtractorTest {
-
- private Activity activity;
- private Set<String> hashtags;
-
- public RegexHashtagExtractorTest(String activityContent, Set<String> hashtags) {
- this.activity = new Activity();
- this.activity.setContent(activityContent);
- this.hashtags = hashtags;
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> params() {
- return Arrays.asList(new Object[][]{
- {"This is the #content of a standard tweet", Stream.of("content").collect(Collectors.toSet())},
- {"This is the content of a standard tweet", new HashSet<>()},
- {"This is the #content of a standard #tweet", Stream.of("content", "tweet").collect(Collectors.toSet())},
- {"UNIX 时间1400000000 秒…… （该睡觉了，各位夜猫子）#程序员#", Stream.of("程序员").collect(Collectors.toSet())},
- {"This is the body of a #fbpost. It can have multiple lines of #content, as well as much more detailed and flowery #language.",
- Stream.of("content", "fbpost", "language").collect(Collectors.toSet())}
- });
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testExtraction() {
- StreamsDatum datum = new StreamsDatum(activity, "Test");
- List<StreamsDatum> result = new RegexHashtagExtractor().process(datum);
- assertThat(result.size(), is(equalTo(1)));
- Activity output = (Activity)result.get(0).getDocument();
- Set<String> extracted = (Set) ExtensionUtil.getInstance().ensureExtensions(output).get(RegexHashtagExtractor.EXTENSION_KEY);
- Set<String> diff = extracted.stream().filter((x) -> !hashtags.contains(x)).collect(Collectors.toSet());
- assertThat(diff.size(), is(equalTo(0)));
- }
-}
diff --git a/streams-contrib/streams-processor-regex/src/test/java/org/apache/streams/regex/RegexMentionExtractorTest.java b/streams-contrib/streams-processor-regex/src/test/java/org/apache/streams/regex/RegexMentionExtractorTest.java
deleted file mode 100644
index 3706b15..0000000
--- a/streams-contrib/streams-processor-regex/src/test/java/org/apache/streams/regex/RegexMentionExtractorTest.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.regex;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-
-import com.google.common.collect.Sets;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-
-@RunWith(Parameterized.class)
-public class RegexMentionExtractorTest {
-
- private Activity activity;
- private Set<Map<String, Object>> mentions;
-
- public RegexMentionExtractorTest(String activityContent, Set<Map<String, Object>> hashtags) {
- this.activity = new Activity();
- this.activity.setContent(activityContent);
- this.mentions = hashtags;
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> params() {
- return Arrays.asList(new Object[][]{
- {"This is the @content of a standard tweet", Stream.of(new HashMap<String, Object>() {{
- put("displayName", "content");
- }}).collect(Collectors.toSet())},
- {"This is the content of a standard tweet", Stream.of(new HashMap<String, Object>()).collect(Collectors.toSet())},
- {"This is the @content of a standard @tweet", Stream.of(new HashMap<String, Object>() {{
- put("displayName", "content");
- }},new HashMap<String, Object>() {{
- put("displayName", "tweet");
- }}).collect(Collectors.toSet())},
- {"UNIX 时间1400000000 秒…… （该睡觉了，各位夜猫子）@程序员#", Stream.of(new HashMap<String, Object>() {{
- put("displayName", "程序员");
- }}).collect(Collectors.toSet())},
- {"This is the body of a @fbpost. It can have multiple lines of #content, as well as much more detailed and flowery @language.",
- Stream.of(new HashMap<String, Object>() {{
- put("displayName", "fbpost");
- }},new HashMap<String, Object>() {{
- put("displayName", "language");
- }}).collect(Collectors.toSet())}
- });
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testExtraction() {
- StreamsDatum datum = new StreamsDatum(activity, "Test");
- List<StreamsDatum> result = new RegexMentionsExtractor().process(datum);
- assertThat(result.size(), is(equalTo(1)));
- Activity output = (Activity)result.get(0).getDocument();
- Set<String> extracted = (Set) ExtensionUtil.getInstance().ensureExtensions(output).get(RegexMentionsExtractor.EXTENSION_KEY);
- Sets.SetView<String> diff = Sets.difference(extracted, mentions);
- assertThat(diff.size(), is(equalTo(0)));
- }
-}
diff --git a/streams-contrib/streams-processor-regex/src/test/java/org/apache/streams/regex/RegexUrlExtractorTest.java b/streams-contrib/streams-processor-regex/src/test/java/org/apache/streams/regex/RegexUrlExtractorTest.java
deleted file mode 100644
index 7c7948b..0000000
--- a/streams-contrib/streams-processor-regex/src/test/java/org/apache/streams/regex/RegexUrlExtractorTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.regex;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.pojo.json.Activity;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-
-@RunWith(Parameterized.class)
-public class RegexUrlExtractorTest {
-
- private Activity activity;
- private Set<String> links;
-
- public RegexUrlExtractorTest(String activityContent, Set<String> links) {
- this.activity = new Activity();
- this.activity.setContent(activityContent);
- this.links = links;
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> params() {
- return Arrays.asList(new Object[][]{
- {"This is the http://t.co/foo of a standard tweet", Stream.of("http://t.co/foo").collect(Collectors.toSet())},
- {"This is the https://t.co/foo of a standard tweet", Stream.of("https://t.co/foo").collect(Collectors.toSet())},
- {"This is the https://t.co/foo of a standard tweet https://t.co/foo", Stream.of("https://t.co/foo").collect(Collectors.toSet())},
- {"This is the http://amd.com/test of a standard tweet", Stream.of("http://amd.com/test").collect(Collectors.toSet())},
- {"This is the content of a standard tweet", new HashSet<>()},
- {"This is the http://www.google.com/articles/awesome?with=query¶ms=true of a standard @tweet",
- Stream.of("http://www.google.com/articles/awesome?with=query¶ms=true").collect(Collectors.toSet())}
- });
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testExtraction() {
- StreamsDatum datum = new StreamsDatum(activity, "Test");
- List<StreamsDatum> result = new RegexUrlExtractor().process(datum);
- assertThat(result.size(), is(equalTo(1)));
- Activity output = (Activity)result.get(0).getDocument();
- Set<String> extracted = new HashSet<>(output.getLinks());
- Set<String> diff = links.stream().filter((x) -> !extracted.contains(x)).collect(Collectors.toSet());
- assertThat(diff.size(), is(equalTo(0)));
- }
-}
diff --git a/streams-contrib/streams-processor-regex/src/test/java/org/apache/streams/regex/RegexUtilsTest.java b/streams-contrib/streams-processor-regex/src/test/java/org/apache/streams/regex/RegexUtilsTest.java
deleted file mode 100644
index a156f3a..0000000
--- a/streams-contrib/streams-processor-regex/src/test/java/org/apache/streams/regex/RegexUtilsTest.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.regex;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-
-@RunWith(Parameterized.class)
-public class RegexUtilsTest {
-
- private final String pattern;
- private final String content;
- private final int wordMatchCount;
- private final int regularMatchCount;
-
- public RegexUtilsTest(String pattern, String content, int regularMatchCount, int wordMatchCount) {
- this.pattern = pattern;
- this.content = content;
- this.wordMatchCount = wordMatchCount;
- this.regularMatchCount = regularMatchCount;
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> parameters() {
- return Arrays.asList(new Object[][]{
- {"[0-9]*", "The number for emergencies is 911.", 1, 1},
- {"#\\w+", "This is#freakingcrazydude.", 1, 0},
- {"#\\w+", "This is #freakingcrazydude.", 1, 1},
- {"#\\w+", "This is #freakingcrazydude!", 1, 1},
- {"#\\w+", "This is #freakingcrazydude I went to the moon", 1, 1},
- {"#\\w+", "This is #freakingcrazydude I went to the #crazy? party", 2, 2},
- {"#\\w+", "This is #freakingcrazydude I went to the #crazy; party", 2, 2},
- {"#\\w+", "This is #freakingcrazydude I went to the #crazy, party", 2, 2},
- {"#\\w+", "This is#freakingcrazydude I went to the #crazy party", 2, 1},
- {"#\\w+", "This is #freakingcrazydude I went to the #freakingcrazydude party", 1, 1},
- {"#\\w+", "#what does the fox say?", 1, 1},
- {"#\\w+", "#what does the fox #say", 2, 2}
- });
- }
-
-
- @Test
- public void testMatches_simple() {
- Map<String, List<Integer>> wordResults = RegexUtils.extractWordMatches(this.pattern, this.content);
- assertThat(wordResults.size(), is(equalTo(wordMatchCount)));
-
- Map<String, List<Integer>> regularResults = RegexUtils.extractMatches(this.pattern, this.content);
- assertThat(regularResults.size(), is(equalTo(regularMatchCount)));
- }
-
-}
diff --git a/streams-contrib/streams-processor-urls/README.md b/streams-contrib/streams-processor-urls/README.md
deleted file mode 100644
index 77861c0..0000000
--- a/streams-contrib/streams-processor-urls/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-processor-urls
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-processor-urls/pom.xml b/streams-contrib/streams-processor-urls/pom.xml
deleted file mode 100644
index b8fcf2b..0000000
--- a/streams-contrib/streams-processor-urls/pom.xml
+++ /dev/null
@@ -1,137 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-processor-urls</artifactId>
- <name>${project.artifactId}</name>
-
- <description>URL Utility Processors</description>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-validator</groupId>
- <artifactId>commons-validator</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.urls</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-</project>
diff --git a/streams-contrib/streams-processor-urls/src/main/java/org/apache/streams/urls/LinkResolver.java b/streams-contrib/streams-processor-urls/src/main/java/org/apache/streams/urls/LinkResolver.java
deleted file mode 100644
index cefac6f..0000000
--- a/streams-contrib/streams-processor-urls/src/main/java/org/apache/streams/urls/LinkResolver.java
+++ /dev/null
@@ -1,450 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.urls;
-
-import org.apache.commons.codec.net.URLCodec;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.validator.routines.UrlValidator;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.net.HttpURLConnection;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-public class LinkResolver implements Serializable {
-
- /**
- * References:
- * Some helpful references to demonstrate the different types of browser re-directs that
- * can happen. If you notice a redirect that was not followed to the proper place please
- * submit a bug at :
- * https://issues.apache.org/jira/browse/STREAMS
- * <p/>
- * Purpose URL
- * ------------- ----------------------------------------------------------------
- * [Status Codes] http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
- * [Test Cases] http://greenbytes.de/tech/tc/httpredirects/
- * [t.co behavior] https://dev.twitter.com/docs/tco-redirection-behavior
- */
-
-
- private final static Logger LOGGER = LoggerFactory.getLogger(LinkResolver.class);
-
- private static final int MAX_ALLOWED_REDIRECTS = 30; // We will only chase the link to it's final destination a max of 30 times.
- private static final int DEFAULT_HTTP_TIMEOUT = 10000; // We will only wait a max of 10,000 milliseconds (10 seconds) for any HTTP response
- private static final String LOCATION_IDENTIFIER = "location";
- private static final String SET_COOKIE_IDENTIFIER = "set-cookie";
-
- // if Bots are not 'ok' this is the spoof settings that we'll use
- private static final Map<String, String> SPOOF_HTTP_HEADERS = new HashMap<String, String>() {{
- put("Connection", "Keep-Alive");
- put("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36");
- put("Accept-Language", "en-US,en;q=0.8,zh;q=0.6");
- }};
-
- // These are the known domains that are 'bot' friendly.
- private static final Collection<String> BOTS_ARE_OK = new ArrayList<String>() {{
- add("t.co");
- }};
-
- // To help canonicalize the URL, these parts are 'known' to be 'ok' to remove
- private static final Collection<String> URL_TRACKING_TO_REMOVE = new ArrayList<String>() {{
- /*
- * Google uses parameters in the URL string to track referrers
- * on their Google Analytics and promotions. These are the
- * identified URL patterns.
- *
- * URL:
- * https://support.google.com/analytics/answer/1033867?hl=en
- *****************************************************************/
-
- // Required. Use utm_source to identify a search engine, newsletter name, or other source.
- add("([\\?&])utm_source(=)[^&?]*");
-
- // Required. Use utm_medium to identify a medium such as email or cost-per- click.
- add("([\\?&])utm_medium(=)[^&?]*");
-
- // Used for paid search. Use utm_term to note the keywords for this ad.
- add("([\\?&])utm_term(=)[^&?]*");
-
- // Used for A/B testing and content-targeted ads. Use utm_content to differentiate ads or links that point to the same
- add("([\\?&])utm_content(=)[^&?]*");
-
- // Used for keyword analysis. Use utm_campaign to identify a specific product promotion or strategic campaign.
- add("([\\?&])utm_campaign(=)[^&?]*");
- }};
-
-
- // This element holds all the information about all the re-directs that have taken place
- // and the steps and HTTP codes that occurred inside of each step.
- private final LinkDetails linkDetails;
- private Collection<String> domainsSensitiveTo = new HashSet<>();
-
- /**
- * Get the link details
- *
- * @return Detailed log of every redirection that took place with the browser along with it it's ultimate status code.
- */
- public LinkDetails getLinkDetails() {
- return linkDetails;
- }
-
- /**
- * Raw string input of the URL. If the URL is invalid, the response code that is returned will indicate such.
- *
- * @param originalURL The URL you wish to unwind represented as a string.
- */
- public LinkResolver(String originalURL) {
- linkDetails = new LinkDetails();
- linkDetails.setOriginalURL(originalURL);
- }
-
- public void run() {
-
- Objects.requireNonNull(linkDetails.getOriginalURL());
-
- linkDetails.setStartTime(DateTime.now());
-
- // we are going to try three times just in case we catch a slow server or one that needs
- // to be warmed up. This tends to happen many times with smaller private servers
- for (int i = 0; (i < 3) && linkDetails.getFinalURL() == null; i++)
- if (linkDetails.getLinkStatus() != LinkDetails.LinkStatus.SUCCESS)
- unwindLink(linkDetails.getOriginalURL());
-
- // because this is a POJO we need to make sure that we set this to false if it was never re-directed
- if(this.linkDetails.getRedirectCount() == 0 || this.linkDetails.getRedirected() == null)
- this.linkDetails.setRedirected(false);
-
- linkDetails.setFinalURL(cleanURL(linkDetails.getFinalURL()));
- if(StringUtils.isNotBlank(linkDetails.getFinalURL()))
- linkDetails.setNormalizedURL(normalizeURL(linkDetails.getFinalURL()));
- if(StringUtils.isNotBlank(linkDetails.getNormalizedURL()))
- linkDetails.setUrlParts(tokenizeURL(linkDetails.getNormalizedURL()));
-
- this.updateTookInMillis();
- }
-
- protected void updateTookInMillis() {
- Objects.requireNonNull(linkDetails.getStartTime());
- linkDetails.setTookInMills(DateTime.now().minus(linkDetails.getStartTime().getMillis()).getMillis());
- }
-
- public void unwindLink(String url) {
- Objects.requireNonNull(linkDetails);
- Objects.requireNonNull(url);
-
- // Check url validity
- UrlValidator urlValidator = new UrlValidator();
- if (!urlValidator.isValid(url)) {
- linkDetails.setLinkStatus(LinkDetails.LinkStatus.MALFORMED_URL);
- return;
- }
-
- // Check to see if they wound up in a redirect loop,
- // IE: 'A' redirects to 'B', then 'B' redirects to 'A'
- if ((linkDetails.getRedirectCount() != null && linkDetails.getRedirectCount() > 0 &&
- (linkDetails.getOriginalURL().equals(url) || linkDetails.getRedirects().contains(url)))
- || (linkDetails.getRedirectCount() != null && linkDetails.getRedirectCount() > MAX_ALLOWED_REDIRECTS)) {
- linkDetails.setLinkStatus(LinkDetails.LinkStatus.LOOP);
- return;
- }
-
- if (!linkDetails.getOriginalURL().equals(url))
- linkDetails.getRedirects().add(url);
-
- HttpURLConnection connection = null;
-
- // Store where the redirected link will go (if there is one)
- String reDirectedLink = null;
-
- try {
- // Turn the string into a URL
- URL thisURL = new URL(url);
-
- // Be sensitive to overloading domains STREAMS-77
- try {
- String host = thisURL.getHost().toLowerCase();
- if(!domainsSensitiveTo.contains(host)) {
- domainsSensitiveTo.add(host);
- long domainWait = LinkResolverHelperFunctions.waitTimeForDomain(thisURL.getHost());
- if (domainWait > 0) {
- LOGGER.debug("Waiting for domain: {}", domainWait);
- Thread.sleep(domainWait);
- }
- }
- } catch(Exception e) {
- // noOp
- }
-
- connection = (HttpURLConnection) new URL(url).openConnection();
-
- // now we are going to pretend that we are a browser...
- // This is the way my mac works.
- if (!BOTS_ARE_OK.contains(thisURL.getHost())) {
- connection.addRequestProperty("Host", thisURL.getHost());
-
- // Bots are not 'ok', so we need to spoof the headers
- for (String k : SPOOF_HTTP_HEADERS.keySet())
- connection.addRequestProperty(k, SPOOF_HTTP_HEADERS.get(k));
-
- // the test to seattlemamadoc.com prompted this change.
- // they auto detect bots by checking the referrer chain and the 'user-agent'
- // this broke the t.co test. t.co URLs are EXPLICITLY ok with bots
- // there is a list for URLS that behave this way at the top in BOTS_ARE_OK
- // smashew 2013-13-2013
- if (linkDetails.getRedirectCount() > 0 && BOTS_ARE_OK.contains(thisURL.getHost()))
- connection.addRequestProperty("Referrer", linkDetails.getOriginalURL());
- }
-
- connection.setReadTimeout(DEFAULT_HTTP_TIMEOUT);
- connection.setConnectTimeout(DEFAULT_HTTP_TIMEOUT);
-
- // we want to follow this behavior on our own to ensure that we are getting to the
- // proper place. This is especially true with links that are wounded by special
- // link winders,
- // IE:
- connection.setInstanceFollowRedirects(false);
-
- if (linkDetails.getCookies() != null)
- for (String cookie : linkDetails.getCookies())
- connection.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
-
- connection.connect();
-
- linkDetails.setFinalResponseCode((long) connection.getResponseCode());
-
- Map<String, List<String>> headers = createCaseInsensitiveMap(connection.getHeaderFields());
- /*
- * If they want us to set cookies, well, then we will set cookies
- * Example URL:
- * http://nyti.ms/1bCpesx
- *****************************************************************/
- if (headers.containsKey(SET_COOKIE_IDENTIFIER))
- linkDetails.getCookies().add(headers.get(SET_COOKIE_IDENTIFIER).get(0));
-
- switch (linkDetails.getFinalResponseCode().intValue()) {
- /*
- * W3C HTTP Response Codes:
- * http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
- */
- case 200: // HTTP OK
- linkDetails.setFinalURL(connection.getURL().toString());
- linkDetails.setDomain(new URL(linkDetails.getFinalURL()).getHost());
- linkDetails.setLinkStatus(LinkDetails.LinkStatus.SUCCESS);
- break;
- case 300: // Multiple choices
- case 301: // URI has been moved permanently
- case 302: // Found
- case 303: // Primarily for a HTTP Post
- case 304: // Not Modified
- case 306: // This status code is unused but in the redirect block.
- case 307: // Temporary re-direct
- /*
- * Author:
- * Smashew
- *
- * Date: 2013-11-15
- *
- * Note:
- * It is possible that we have already found our final URL. In
- * the event that we have found our final URL, we are going to
- * save this URL as long as it isn't the original URL.
- * We are still going to ask the browser to re-direct, but in the
- * case of yet another redirect, seen with the redbull test
- * this can be followed by a 304, a browser, by W3C standards would
- * still render the page with it's content, but for us to assert
- * a success, we are really hoping for a 304 message.
- *******************************************************************/
- if (!linkDetails.getOriginalURL().toLowerCase().equals(connection.getURL().toString().toLowerCase()))
- linkDetails.setFinalURL(connection.getURL().toString());
- if (!headers.containsKey(LOCATION_IDENTIFIER)) {
- LOGGER.info("Headers: {}", headers);
- linkDetails.setLinkStatus(LinkDetails.LinkStatus.REDIRECT_ERROR);
- } else {
- linkDetails.setRedirected(Boolean.TRUE);
- linkDetails.setRedirectCount(linkDetails.getRedirectCount() + 1);
- reDirectedLink = connection.getHeaderField(LOCATION_IDENTIFIER);
- }
- break;
- case 305: // User must use the specified proxy (deprecated by W3C)
- break;
- case 401: // Unauthorized (nothing we can do here)
- linkDetails.setLinkStatus(LinkDetails.LinkStatus.UNAUTHORIZED);
- break;
- case 403: // HTTP Forbidden (Nothing we can do here)
- linkDetails.setLinkStatus(LinkDetails.LinkStatus.FORBIDDEN);
- break;
- case 404: // Not Found (Page is not found, nothing we can do with a 404)
- linkDetails.setLinkStatus(LinkDetails.LinkStatus.NOT_FOUND);
- break;
- case 500: // Internal Server Error
- case 501: // Not Implemented
- case 502: // Bad Gateway
- case 503: // Service Unavailable
- case 504: // Gateway Timeout
- case 505: // Version not supported
- linkDetails.setLinkStatus(LinkDetails.LinkStatus.HTTP_ERROR_STATUS);
- break;
- default:
- LOGGER.info("Unrecognized HTTP Response Code: {}", linkDetails.getFinalResponseCode());
- linkDetails.setLinkStatus(LinkDetails.LinkStatus.NOT_FOUND);
- break;
- }
- } catch (MalformedURLException e) {
- // the URL is trash, so, it can't load it.
- linkDetails.setLinkStatus(LinkDetails.LinkStatus.MALFORMED_URL);
- } catch (IOException ex) {
- // there was an issue we are going to set to error.
- linkDetails.setLinkStatus(LinkDetails.LinkStatus.ERROR);
- } catch (Exception ex) {
- // there was an unknown issue we are going to set to exception.
- linkDetails.setLinkStatus(LinkDetails.LinkStatus.EXCEPTION);
- } finally {
- // if the connection is not null, then we need to disconnect to close any underlying resources
- if (connection != null)
- connection.disconnect();
- }
-
- // If there was a redirection, then we have to keep going
- // Placing this code here should help to satisfy ensuring that the connection object
- // is closed successfully.
- if (reDirectedLink != null)
- unwindLink(reDirectedLink);
-
- }
-
- private Map<String, List<String>> createCaseInsensitiveMap(Map<String, List<String>> input) {
- Map<String, List<String>> toReturn = new HashMap<>();
- for (String k : input.keySet())
- if (k != null && input.get(k) != null)
- toReturn.put(k.toLowerCase(), input.get(k));
- return toReturn;
- }
-
- private String cleanURL(String url) {
- // If they pass us a null URL then we are going to pass that right back to them.
- if (url == null)
- return null;
-
- // remember how big the URL was at the start
- int startLength = url.length();
-
- // Iterate through all the known URL parameters of tracking URLs
- for (String pattern : URL_TRACKING_TO_REMOVE)
- url = url.replaceAll(pattern, "");
-
- // If the URL is smaller than when it came in. Then it had tracking information
- if (url.length() < startLength)
- linkDetails.setTracked(Boolean.TRUE);
-
- // return our url.
- return url;
- }
-
- /**
- * Removes the protocol, if it exists, from the front and
- * removes any random encoding characters
- * Extend this to do other url cleaning/pre-processing
- *
- * @param url - The String URL to normalize
- * @return normalizedUrl - The String URL that has no junk or surprises
- */
- public static String normalizeURL(String url) {
- // Decode URL to remove any %20 type stuff
- String normalizedUrl = url;
- try {
-
- // Replaced URLDecode with commons-codec b/c of failing tests
-
- URLCodec codec = new URLCodec();
-
- normalizedUrl = codec.decode(url);
-
- // Remove the protocol, http:// ftp:// or similar from the front
- if (normalizedUrl.contains("://"))
- normalizedUrl = normalizedUrl.split(":/{2}")[1];
-
- } catch (NullPointerException npe) {
- System.err.println("NPE Decoding URL. Decoding skipped.");
- npe.printStackTrace();
- } catch (Throwable e) {
- System.err.println("Misc error Decoding URL. Decoding skipped.");
- e.printStackTrace();
- }
-
-
- // Room here to do more pre-processing
-
- return normalizedUrl;
- }
-
- /**
- * Goal is to get the different parts of the URL path. This can be used
- * in a classifier to help us determine if we are working with
- * <p/>
- * Reference:
- * http://stackoverflow.com/questions/10046178/pattern-matching-for-url-classification
- *
- * @param url - Url to be tokenized
- * @return tokens - A String array of all the tokens
- */
- public static List<String> tokenizeURL(String url) {
- url = normalizeURL(url);
- // I assume that we're going to use the whole URL to find tokens in
- // If you want to just look in the GET parameters, or you want to ignore the domain
- // or you want to use the domain as a token itself, that would have to be
- // processed above the next line, and only the remaining parts split
- List<String> toReturn = new ArrayList<>();
-
- // Split the URL by forward slashes. Most modern browsers will accept a URL
- // this malformed such as http://www.smashew.com/hello//how////are/you
- // hence the '+' in the regular expression.
- for (String part : url.split("/+"))
- toReturn.add(part.toLowerCase());
-
- // return our object.
- return toReturn;
-
- // One could alternatively use a more complex regex to remove more invalid matches
- // but this is subject to your (?:in)?ability to actually write the regex you want
-
- // These next two get rid of tokens that are too short, also.
-
- // Destroys anything that's not alphanumeric and things that are
- // alphanumeric but only 1 character long
- //String[] tokens = url.split("(?:[\\W_]+\\w)*[\\W_]+");
-
- // Destroys anything that's not alphanumeric and things that are
- // alphanumeric but only 1 or 2 characters long
- //String[] tokens = url.split("(?:[\\W_]+\\w{1,2})*[\\W_]+");
- }
-
-
-}
diff --git a/streams-contrib/streams-processor-urls/src/main/java/org/apache/streams/urls/LinkResolverHelperFunctions.java b/streams-contrib/streams-processor-urls/src/main/java/org/apache/streams/urls/LinkResolverHelperFunctions.java
deleted file mode 100644
index 5493fef..0000000
--- a/streams-contrib/streams-processor-urls/src/main/java/org/apache/streams/urls/LinkResolverHelperFunctions.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.urls;
-
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.regex.Pattern;
-
-/**
- * This is a static utility helper class to verify strings are URLs,
- * obey domain sensitivity, and find URLs within a string.
- * It cannot be, instantiated and can only be referenced through
- * the static accessor functions
- *
- */
-public final class LinkResolverHelperFunctions {
-
- private static final String REGEX_URL =
- "(?:(?:https?|ftp)://)" + // protocol identifier
- "(?:\\S+(?::\\S*)?@)?" + // user:pass authentication
- "(?:" +
- "(?!(?:10|127)(?:\\.\\d{1,3}){3})" + // IP address exclusion
- "(?!(?:169\\.254|192\\.168)(?:\\.\\d{1,3}){2})" + // private & local networks
- "(?!172\\.(?:1[6-9]|2\\d|3[0-1])(?:\\.\\d{1,3}){2})" +
- "(?:[1-9]\\d?|1\\d\\d|2[01]\\d|22[0-3])" + // IP address dotted notation octets
- "(?:\\.(?:1?\\d{1,2}|2[0-4]\\d|25[0-5])){2}" + // excludes loop-back network 0.0.0.0, excludes network & broadcast addresses
- "(?:\\.(?:[1-9]\\d?|1\\d\\d|2[0-4]\\d|25[0-4]))" + // excludes reserved space >= 224.0.0.0, (first & last IP address of each class)
- "|" +
- "(?:(?:[a-z\\u00a1-\\uffff0-9]+-?)*[a-z\\u00a1-\\uffff0-9]+)" + // host name
- "(?:\\.(?:[a-z\\u00a1-\\uffff0-9]+-?)*[a-z\\u00a1-\\uffff0-9]+)*" + // domain name
- "(?:\\.(?:[a-z\\u00a1-\\uffff]{2,}))" + // TLD identifier
- ")" +
- "(?::\\d{2,5})?" + // port number
- "(?:/[^\\s]*)?"; // resource path
-
- private static final String REGEX_URL_EXPLICIT = "^" + REGEX_URL + "$";
-
- // The amount of time we want to space between domain calls
- public static final long RECENT_DOMAINS_BACKOFF = 1000;
- public static final long DEFAULT_STAGGER = RECENT_DOMAINS_BACKOFF / 10;
-
- // Map to store the information of recent domains, with the last time they were accessed.
- private static final ConcurrentMap<String, Date> RECENT_DOMAINS = new ConcurrentHashMap<>();
-
- private static Timer timer;
-
- /**
- * Check to see if this string is a URL or not
- * @param possibleURL
- * The possible URL that we would like to test
- * @return
- * Whether or not it is a URL
- */
- public static boolean isURL(String possibleURL) {
- return possibleURL.matches(REGEX_URL_EXPLICIT);
- }
-
- public static boolean containsURLs(String possiblyHasURLs) {
- return possiblyHasURLs != null && Pattern.compile(REGEX_URL).matcher(possiblyHasURLs).find();
- }
-
- private LinkResolverHelperFunctions() {
- // force it not to be instantiated.
- }
-
- public static void purgeAllDomainWaitTimes() {
- RECENT_DOMAINS.clear();
- }
-
- public static long waitTimeForDomain(String domain) {
- domain = domain.toLowerCase();
-
- long toReturn = 0;
- synchronized (LinkResolverHelperFunctions.class) {
- purgeAnyExpiredDomains();
- // if the timer doesn't exist, then setup the timer (IE: first time called)
- if(timer == null)
- setupTimer();
- long currentTime = new Date().getTime();
-
- if(RECENT_DOMAINS.containsKey(domain)) {
- // find the time it wants us to wait until
- long nextTime = RECENT_DOMAINS.get(domain).getTime();
- long random = (long)((Math.random() * (RECENT_DOMAINS_BACKOFF / 5))); // stagger
-
- // back-off has been satisfied
- if(currentTime >= nextTime)
- RECENT_DOMAINS.put(domain, new Date(currentTime + RECENT_DOMAINS_BACKOFF));
- else {
- // we are going to have to wait longer than the back-off
- // add the time we told them they needed to wait
- toReturn = (nextTime - currentTime) + RECENT_DOMAINS_BACKOFF;
- RECENT_DOMAINS.put(domain, new Date(currentTime + toReturn));
- toReturn += random + 1;
- }
- } else {
- // no wait
- RECENT_DOMAINS.put(domain, new Date(currentTime + RECENT_DOMAINS_BACKOFF));
- }
- } // end synchronized block
-
- return toReturn;
- }
-
- /**
- * Quick function to setup the daemon to clear domains to keep our memory foot-print low
- */
- private static void setupTimer() {
- timer = new Timer(true);
- timer.schedule(new TimerTask() {
- public void run() {
- purgeAnyExpiredDomains();
- }
- }, RECENT_DOMAINS_BACKOFF * 2);
- }
-
- /**
- * called by the timer to expire any domains
- */
- private static void purgeAnyExpiredDomains() {
- // ensure this method is synchronized to get the proper information
- synchronized (LinkResolverHelperFunctions.class) {
- // figure the time that we would like for these domains to expire
- long currentTime = new Date().getTime();
- // see if there is any work that 'can' be done
- if(RECENT_DOMAINS.size() != 0) {
- // create a temporary list of the items that can be removed
- Collection<String> ableToRemove = new HashSet<>();
-
-
- // iterate through all the domains (keys)
- // if it qualifies, we can add it to the remove list
- for(String k : RECENT_DOMAINS.keySet())
- if(currentTime >= RECENT_DOMAINS.get(k).getTime())
- ableToRemove.add(k);
-
- if(ableToRemove.size() > 0) // if there are domains to remove, then remove them
- for(String k : ableToRemove) // iterate through every domain that we can remove
- RECENT_DOMAINS.remove(k); // remove the domain from our map.
- }
- }
- }
-
-}
diff --git a/streams-contrib/streams-processor-urls/src/main/java/org/apache/streams/urls/LinkResolverProcessor.java b/streams-contrib/streams-processor-urls/src/main/java/org/apache/streams/urls/LinkResolverProcessor.java
deleted file mode 100644
index d13d80b..0000000
--- a/streams-contrib/streams-processor-urls/src/main/java/org/apache/streams/urls/LinkResolverProcessor.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.urls;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public class LinkResolverProcessor implements StreamsProcessor {
-
- private static final String STREAMS_ID = "LinkResolverProcessor";
- private static final Logger LOGGER = LoggerFactory.getLogger(LinkResolverProcessor.class);
- private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- List<StreamsDatum> result = new ArrayList<>();
-
- LOGGER.debug("{} processing {}", STREAMS_ID, entry.getDocument().getClass());
-
- Activity activity;
-
- // get list of shared urls
- if (entry.getDocument() instanceof Activity) {
- activity = (Activity) entry.getDocument();
-
- activity.setLinks(new ArrayList<>(unwind(activity.getLinks())));
-
- entry.setDocument(activity);
-
- result.add(entry);
-
- return result;
- } else if (entry.getDocument() instanceof String) {
-
- try {
- activity = mapper.readValue((String) entry.getDocument(), Activity.class);
- } catch (Exception e) {
- e.printStackTrace();
- LOGGER.warn(e.getMessage());
- return Stream.of(entry).collect(Collectors.toList());
- }
-
- activity.setLinks(new ArrayList<>(unwind(activity.getLinks())));
-
- try {
- entry.setDocument(mapper.writeValueAsString(activity));
- } catch (Exception e) {
- e.printStackTrace();
- LOGGER.warn(e.getMessage());
- return new ArrayList<>();
- }
-
- result.add(entry);
-
- return result;
-
- } else {
- return new ArrayList<>();
- }
- }
-
- @Override
- public void prepare(Object o) {
- // noOp
- }
-
- @Override
- public void cleanUp() {
- // noOp
- }
-
-
- protected Set<String> unwind(List<String> inputLinks) {
- Set<String> outputLinks = new HashSet<>();
- for (String link : inputLinks) {
- try {
- LinkResolver unwinder = new LinkResolver(link);
- unwinder.run();
- outputLinks.add(unwinder.getLinkDetails().getFinalURL());
- } catch (Exception e) {
- //if unwindable drop
- LOGGER.debug("Failed to unwind link : {}", link);
- LOGGER.debug("Exception unwinding link : {}", e);
- e.printStackTrace();
- }
- }
- return outputLinks;
- }
-}
diff --git a/streams-contrib/streams-processor-urls/src/main/jsonschema/org/apache/streams/urls/LinkDetails.json b/streams-contrib/streams-processor-urls/src/main/jsonschema/org/apache/streams/urls/LinkDetails.json
deleted file mode 100644
index 21099d7..0000000
--- a/streams-contrib/streams-processor-urls/src/main/jsonschema/org/apache/streams/urls/LinkDetails.json
+++ /dev/null
@@ -1,78 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.urls.LinkDetails",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "linkStatus" : {
- "type" : "string",
- "enum" : [
- "SUCCESS",
- "ERROR",
- "MALFORMED_URL",
- "NOT_FOUND",
- "FORBIDDEN",
- "REDIRECT_ERROR",
- "UNAUTHORIZED",
- "LOOP",
- "HTTP_ERROR_STATUS",
- "EXCEPTION"
- ]
- },
- "originalURL": {
- "type": "string"
- },
- "finalURL": {
- "type": "string"
- },
- "normalizedURL": {
- "type": "string"
- },
- "urlParts": {
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "domain": {
- "type": "string"
- },
- "redirected": {
- "type": "boolean"
- },
- "redirects": {
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "redirectCount": {
- "type": "integer",
- "default": 0
- },
- "tracked": {
- "type": "boolean"
- },
- "finalResponseCode" : {
- "type" : "integer"
- },
- "cookies": {
- "type": "array",
- "uniqueItems": true,
- "items": {
- "type": "string"
- }
- },
- "startTime": {
- "type": "string",
- "format" : "date-time"
- },
- "tookInMills": {
- "type": "integer"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-processor-urls/src/main/resources/components.dot b/streams-contrib/streams-processor-urls/src/main/resources/components.dot
deleted file mode 100644
index 161a192..0000000
--- a/streams-contrib/streams-processor-urls/src/main/resources/components.dot
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- generators [label="generators", shape="circle"];
- providers [label="providers", shape="circle"];
- persisters [label="persisters", shape="circle"];
-
- subgraph cluster_processors {
- label="processors";
- processors_urls_LinkResolverProcessor [label="LinkResolverProcessor"]
- }
-
- generators -> providers
-
- providers -> processors_urls_LinkResolverProcessor [label="StreamsDatum[Activity]\nStreamsDatum[String]"]
- processors_urls_LinkResolverProcessor -> persisters [label="StreamsDatum[Activity]\nStreamsDatum[ObjectNode]"]
-
-}
diff --git a/streams-contrib/streams-processor-urls/src/site/markdown/index.md b/streams-contrib/streams-processor-urls/src/site/markdown/index.md
deleted file mode 100644
index b2b4c02..0000000
--- a/streams-contrib/streams-processor-urls/src/site/markdown/index.md
+++ /dev/null
@@ -1,16 +0,0 @@
-streams-processor-urls
-=====================
-
-URL Utility Processors
-
-## Components
-
-
-
-| Class |
-|-------|
-| LinkResolverProcessor [LinkResolverProcessor.html](apidocs/org/apache/streams/urls/LinkResolverProcessor.html "javadoc")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-processor-urls/src/test/java/org/apache/streams/urls/LinkHelperFunctionsTest.java b/streams-contrib/streams-processor-urls/src/test/java/org/apache/streams/urls/LinkHelperFunctionsTest.java
deleted file mode 100644
index 6c8439f..0000000
--- a/streams-contrib/streams-processor-urls/src/test/java/org/apache/streams/urls/LinkHelperFunctionsTest.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.urls;
-
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Date;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class LinkHelperFunctionsTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(LinkHelperFunctionsTest.class);
-
- @Test
- public void testIsURL() {
- assertTrue(LinkResolverHelperFunctions.isURL("http://goo.gl/wSrHDA"));
- assertTrue(LinkResolverHelperFunctions.isURL("http://ow.ly/u4Kte"));
- assertTrue(LinkResolverHelperFunctions.isURL("http://x.co/3yapt"));
- assertTrue(LinkResolverHelperFunctions.isURL("http://bit.ly/1cX5Rh4"));
- assertTrue(LinkResolverHelperFunctions.isURL("http://t.co/oP8JYB0MYW"));
- assertTrue(LinkResolverHelperFunctions.isURL("http://goo.gl/wSrHDA"));
- assertTrue(LinkResolverHelperFunctions.isURL("http://t.co/fBoCby3l1t"));
- assertTrue(LinkResolverHelperFunctions.isURL("http://paper.li/GuyKawasaki"));
- assertTrue(LinkResolverHelperFunctions.isURL("http://www.google.com"));
- assertTrue(LinkResolverHelperFunctions.isURL("http://goo.gl/wSrHDA"));
- assertTrue(LinkResolverHelperFunctions.isURL("http://www.cnn.com"));
- }
-
- @Test
- public void testContainsURL() {
- assertTrue(LinkResolverHelperFunctions.containsURLs("here is the URL: http://goo.gl/wSrHDA"));
- assertTrue(LinkResolverHelperFunctions.containsURLs("a lovely day for URLing it up http://ow.ly/u4Kte"));
- assertTrue(LinkResolverHelperFunctions.containsURLs("http://x.co/3yapt is really cool"));
- assertTrue(LinkResolverHelperFunctions.containsURLs("http://bit.ly/1cX5Rh4 me likes"));
- assertTrue(LinkResolverHelperFunctions.containsURLs("http://t.co/oP8JYB0MYW wtf mate?"));
- assertTrue(LinkResolverHelperFunctions.containsURLs("Every morning is a good morning in URL world: http://goo.gl/wSrHDA"));
-
- assertFalse(LinkResolverHelperFunctions.containsURLs("Every day I awake, only to find, I have no URLS"));
- assertFalse(LinkResolverHelperFunctions.containsURLs("Http:// to be or not to be"));
- assertFalse(LinkResolverHelperFunctions.containsURLs("Can I get an http://X up on the board pat?"));
- assertFalse(LinkResolverHelperFunctions.containsURLs("You must remember Joey, no matter how much you ftp://stink you must never, EVER, take a shower in my dressing room!"));
- }
-
-
- @Test
- public void testSimple() {
-
- LinkResolverHelperFunctions.purgeAllDomainWaitTimes();
- String domain1 = "smashew.com";
-
- // safe to run...
- assertEquals("smashew.com: No need to wait", 0, LinkResolverHelperFunctions.waitTimeForDomain(domain1));
- // get required sleep
- long smashewSleepTime1 = LinkResolverHelperFunctions.waitTimeForDomain(domain1);
- // sleep
- LOGGER.debug("Sleeping: " + new Date().getTime() + "-" + smashewSleepTime1);
- safeSleep(smashewSleepTime1);
- LOGGER.debug("Slept For: " + new Date().getTime() + "-" + smashewSleepTime1);
- // safe to run again
- assertEquals("smashew.com: No need to wait", 0, LinkResolverHelperFunctions.waitTimeForDomain(domain1));
- }
-
- private static void safeSleep(long millis) {
- try {
- Thread.sleep(millis);
- }
- catch(Exception e) {
- // noOp
- }
- }
-
- @Test
- public void testSingle() {
-
- LinkResolverHelperFunctions.purgeAllDomainWaitTimes();
-
- String domain1 = "smashew.com";
- String domain2 = "google.com";
-
- long smashewSleepTime1 = LinkResolverHelperFunctions.waitTimeForDomain(domain1);
- long smashewSleepTime2 = LinkResolverHelperFunctions.waitTimeForDomain(domain1);
- long smashewSleepTime3 = LinkResolverHelperFunctions.waitTimeForDomain(domain1);
- long smashewSleepTime4 = LinkResolverHelperFunctions.waitTimeForDomain(domain1);
-
- LOGGER.debug("smashew.com: " + smashewSleepTime1 + "," + smashewSleepTime2 + "," + smashewSleepTime3 + "," + smashewSleepTime4);
-
- assertEquals("smashew.com: No need to wait", 0, smashewSleepTime1);
- assertTrue("smashew.com: Wait for at least min x 1", smashewSleepTime2 >= (LinkResolverHelperFunctions.RECENT_DOMAINS_BACKOFF - LinkResolverHelperFunctions.DEFAULT_STAGGER));
- assertTrue("smashew.com: Wait for at least min x 2", smashewSleepTime3 >= (LinkResolverHelperFunctions.RECENT_DOMAINS_BACKOFF * 2) - (LinkResolverHelperFunctions.DEFAULT_STAGGER * 2));
- assertTrue("smashew.com: Wait for at least min x 3", smashewSleepTime4 >= (LinkResolverHelperFunctions.RECENT_DOMAINS_BACKOFF * 3) - (LinkResolverHelperFunctions.DEFAULT_STAGGER * 3));
-
- long timeBeforeSleep = new Date().getTime();
- LOGGER.debug("Sleeping for: " + smashewSleepTime4 + " ms");
-
- safeSleep(smashewSleepTime4);
- LOGGER.debug("Actually slept for: " + (new Date().getTime() - timeBeforeSleep) + " ms");
-
- long postSleepDomain1 = LinkResolverHelperFunctions.waitTimeForDomain(domain1);
- LOGGER.debug("smashew.com: Post Sleep domain1: " + postSleepDomain1);
- assertEquals("Smashew.com: No need to wait after sleep", 0, postSleepDomain1);
-
- }
-
- @Test
- public void testMulti() {
-
- LinkResolverHelperFunctions.purgeAllDomainWaitTimes();
- String domain1 = "smashew.com";
- String domain2 = "google.com";
-
- long smashewSleepTime1 = LinkResolverHelperFunctions.waitTimeForDomain(domain1);
- long smashewSleepTime2 = LinkResolverHelperFunctions.waitTimeForDomain(domain1);
- long smashewSleepTime3 = LinkResolverHelperFunctions.waitTimeForDomain(domain1);
-
- long googleSleepTime1 = LinkResolverHelperFunctions.waitTimeForDomain(domain2);
- long googleSleepTime2 = LinkResolverHelperFunctions.waitTimeForDomain(domain2);
-
- LOGGER.debug("smashew.com: " + smashewSleepTime1 + "," + smashewSleepTime2 + "," + smashewSleepTime3);
- LOGGER.debug("google.com: " + googleSleepTime1 + "," + googleSleepTime2);
-
- assertEquals("smashew.com: No need to wait", 0, smashewSleepTime1);
- assertTrue("smashew.com: Wait for at least min x 1", smashewSleepTime2 >= (LinkResolverHelperFunctions.RECENT_DOMAINS_BACKOFF - LinkResolverHelperFunctions.DEFAULT_STAGGER));
- assertTrue("smashew.com: Wait for at least min x 2", smashewSleepTime3 >= (LinkResolverHelperFunctions.RECENT_DOMAINS_BACKOFF * 2) - (LinkResolverHelperFunctions.DEFAULT_STAGGER * 2));
-
- assertEquals("google.com: No need to wait", 0, googleSleepTime1);
- assertTrue("google.com: No need to wait", googleSleepTime2 >= LinkResolverHelperFunctions.RECENT_DOMAINS_BACKOFF - LinkResolverHelperFunctions.DEFAULT_STAGGER);
-
- try {
- LOGGER.debug("WAITING FOR: " + smashewSleepTime3);
- Thread.sleep(smashewSleepTime3);
- }
- catch(Exception e) {
- // noOp
- }
-
- long postSleepDomain1 = LinkResolverHelperFunctions.waitTimeForDomain(domain1);
- long postSleepDomain2 = LinkResolverHelperFunctions.waitTimeForDomain(domain2);
-
- LOGGER.debug("smashew.com: Post Sleep domain1: " + postSleepDomain1);
- LOGGER.debug("google.com: Post Sleep domain2: " + postSleepDomain2);
-
- assertEquals("Smashew.com: No need to wait after sleep", 0, postSleepDomain1);
- assertEquals("google.com: No need to wait after sleep", 0, postSleepDomain2);
-
- }
-
-}
diff --git a/streams-contrib/streams-processor-urls/src/test/java/org/apache/streams/urls/TestLinkUnwinderProcessor.java b/streams-contrib/streams-processor-urls/src/test/java/org/apache/streams/urls/TestLinkUnwinderProcessor.java
deleted file mode 100644
index 3af3a89..0000000
--- a/streams-contrib/streams-processor-urls/src/test/java/org/apache/streams/urls/TestLinkUnwinderProcessor.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.urls;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import org.apache.commons.lang3.SerializationUtils;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-public class TestLinkUnwinderProcessor {
-
- private static String activityString;
-
- @Test
- public void testLinkUnwinderBadDomain() {
- LinkResolver resolver = new LinkResolver("http://nope@#$%");
- resolver.run();
- LinkDetails details = resolver.getLinkDetails();
- assertEquals("Should be 400", LinkDetails.LinkStatus.MALFORMED_URL, details.getLinkStatus());
- }
-
- @Ignore
- @Test
- public void testLinkResolverSerializability() {
- LinkResolver resolver = new LinkResolver("http://bit.ly/1cX5Rh4");
- LinkResolver clone = SerializationUtils.clone(resolver);
- }
-
- @Ignore
- @Test
- public void test404Link() {
- LinkResolver resolver = new LinkResolver("http://www.kneesupmotherbrown.me/2013/05/26/well-its-fair-to-say-may-has-been-a-crappy-month");
- resolver.run();
- LinkDetails details = resolver.getLinkDetails();
- assertEquals("Should be 404", LinkDetails.LinkStatus.NOT_FOUND, details.getLinkStatus());
- }
-
- @Test
- public void testLinkResolverProcessorSerializability() {
- LinkResolverProcessor processor = new LinkResolverProcessor();
- LinkResolverProcessor clone = SerializationUtils.clone(processor);
- }
-
- @Ignore
- @Test
- public void testActivityLinkUnwinderProcessorBitly() throws Exception{
- testActivityUnwinderHelper(Collections.singletonList("http://bit.ly/1cX5Rh4"), Collections.singletonList("http://www.wcgworld.com/"));
- testStringActivityUnwinderHelper(Collections.singletonList("http://bit.ly/1cX5Rh4"), Collections.singletonList("http://www.wcgworld.com/"));
- }
-
- @Ignore
- @Test
- public void testActivityLinkUnwinderProcessorTdotCo() throws Exception{
- testActivityUnwinderHelper(Collections.singletonList("http://t.co/lLFgFynv2G"), Collections.singletonList("http://www.holmesreport.com/latest"));
- testStringActivityUnwinderHelper(Collections.singletonList("http://t.co/lLFgFynv2G"), Collections.singletonList("http://www.holmesreport.com/latest"));
- }
-
- @Ignore
- @Test
- public void testActivityLinkUnwinderProcessorGoogle() throws Exception{
- testActivityUnwinderHelper(Collections.singletonList("http://goo.gl/wSrHDA"), Collections.singletonList("http://www.wcgworld.com/"));
- testStringActivityUnwinderHelper(Collections.singletonList("http://goo.gl/wSrHDA"), Collections.singletonList("http://www.wcgworld.com/"));
- }
-
- @Ignore
- @Test
- public void testActivityLinkUnwinderProcessorOwly() throws Exception{
- testActivityUnwinderHelper(Collections.singletonList("http://ow.ly/u4Kte"), Collections.singletonList("http://www.wcgworld.com/"));
- testStringActivityUnwinderHelper(Collections.singletonList("http://ow.ly/u4Kte"), Collections.singletonList("http://www.wcgworld.com/"));
- }
-
- @Ignore
- @Test
- public void testActivityLinkUnwinderProcessorGoDaddy() throws Exception{
- testActivityUnwinderHelper(Collections.singletonList("http://x.co/3yapt"), Collections.singletonList("http://www.wcgworld.com/"));
- testStringActivityUnwinderHelper(Collections.singletonList("http://x.co/3yapt"), Collections.singletonList("http://www.wcgworld.com/"));
- }
-
- @Ignore
- @Test
- public void testActivityLinkUnwinderProcessorMulti() throws Exception{
- // changed these tests because the processor now guarantees each result returned only once
- testActivityUnwinderHelper(Arrays.asList("http://x.co/3yapt", "http://ow.ly/u4Kte", "http://goo.gl/wSrHDA"), Collections.singletonList("http://www.wcgworld.com/"));
- testStringActivityUnwinderHelper(Arrays.asList("http://x.co/3yapt", "http://ow.ly/u4Kte", "http://goo.gl/wSrHDA"), Collections.singletonList("http://www.wcgworld.com/"));
- }
-
- public void testActivityUnwinderHelper(List<String> input, List<String> expected) throws Exception{
-
- // Purge all of the domain wait times (for testing)
- LinkResolverHelperFunctions.purgeAllDomainWaitTimes();
-
- // Create a new activity
- Activity activity = new Activity();
- activity.setLinks(input);
- StreamsDatum datum = new StreamsDatum(activity);
- LinkResolverProcessor processor = new LinkResolverProcessor();
- processor.prepare(null);
-
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertTrue(resultDatum.getDocument() instanceof Activity);
-
-
- Activity resultActivity = (Activity) resultDatum.getDocument();
- assertNotNull(resultActivity.getLinks());
- List<String> resultLinks = resultActivity.getLinks();
- assertEquals(expected.size(), resultLinks.size());
- assertEquals(expected, resultLinks);
- }
-
- public void testStringActivityUnwinderHelper(List<String> input, List<String> expected) throws Exception{
- LinkResolverHelperFunctions.purgeAllDomainWaitTimes();
- Activity activity = new Activity();
- activity.setLinks(input);
- String str = StreamsJacksonMapper.getInstance().writeValueAsString(activity);
- StreamsDatum datum = new StreamsDatum(str);
- LinkResolverProcessor processor = new LinkResolverProcessor();
- processor.prepare(null);
- List<StreamsDatum> result = processor.process(datum);
- assertNotNull(result);
- assertEquals(1, result.size());
- StreamsDatum resultDatum = result.get(0);
- assertNotNull(resultDatum);
- assertTrue(resultDatum.getDocument() instanceof String);
- String resultActivityString = (String) resultDatum.getDocument();
- Activity resultActivity = StreamsJacksonMapper.getInstance().readValue(resultActivityString, Activity.class);
- assertNotNull(resultActivity.getLinks());
- List<String> resultLinks = resultActivity.getLinks();
- assertEquals(expected.size(), resultLinks.size());
- assertEquals(expected, resultLinks);
- }
-
-}
diff --git a/streams-contrib/streams-provider-facebook/README.md b/streams-contrib/streams-provider-facebook/README.md
deleted file mode 100644
index ad58d68..0000000
--- a/streams-contrib/streams-provider-facebook/README.md
+++ /dev/null
@@ -1,17 +0,0 @@
-streams-provider-facebook
-=====================
-
-Facebook Provider
-
-Example configuration:
-
- facebook {
- oauth {
- appId = ""
- appSecret = ""
- accessToken = ""
- }
- }
-
-
-
diff --git a/streams-contrib/streams-provider-facebook/metadata/facebook_post.mup b/streams-contrib/streams-provider-facebook/metadata/facebook_post.mup
deleted file mode 100644
index 9668633..0000000
--- a/streams-contrib/streams-provider-facebook/metadata/facebook_post.mup
+++ /dev/null
@@ -1,697 +0,0 @@
-{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "title": "Serializer",
- "id": 1,
- "formatVersion": 2,
- "attr": {
- "style": {}
- },
- "ideas": {
- "1": {
- "title": "Activity",
- "id": 2,
- "ideas": {
- "1": {
- "title": "actor",
- "id": 4,
- "ideas": {
- "1": {
- "title": "summary",
- "id": 28
- },
- "7": {
- "title": "url",
- "id": 41
- },
- "0.5": {
- "title": "displayName",
- "id": 32,
- "attr": {
- "style": {}
- }
- },
- "0.75": {
- "title": "extensions",
- "id": 33,
- "ideas": {
- "1": {
- "title": "followers",
- "id": 34
- },
- "2": {
- "title": "handle",
- "id": 35
- },
- "3": {
- "title": "location",
- "id": 36
- },
- "4": {
- "title": "location",
- "id": 37
- },
- "5": {
- "title": "screenName",
- "id": 38
- },
- "6": {
- "title": "posts",
- "id": 39
- },
- "7": {
- "title": "favorites",
- "id": 40
- }
- },
- "attr": {
- "style": {}
- }
- },
- "0.875": {
- "title": "handle",
- "id": 30,
- "attr": {
- "style": {}
- }
- },
- "0.9375": {
- "title": "id",
- "id": 29,
- "attr": {
- "style": {}
- }
- },
- "0.96875": {
- "title": "image",
- "id": 31,
- "attr": {
- "style": {}
- }
- }
- }
- },
- "2": {
- "title": "id",
- "id": 5
- },
- "3": {
- "title": "verb",
- "id": 6
- },
- "1.5": {
- "title": "content",
- "id": 16,
- "attr": {
- "style": {}
- }
- },
- "2.5": {
- "title": "icon",
- "id": 13,
- "attr": {
- "style": {}
- }
- },
- "1.75": {
- "title": "generator",
- "id": 12,
- "attr": {
- "style": {}
- },
- "ideas": {
- "1": {
- "title": "id",
- "id": 62
- },
- "0.5": {
- "title": "displayName",
- "id": 63,
- "attr": {
- "style": {}
- }
- }
- }
- },
- "2.75": {
- "title": "links",
- "id": 18,
- "attr": {
- "style": {}
- }
- },
- "2.875": {
- "title": "object",
- "id": 8,
- "attr": {
- "style": {}
- },
- "ideas": {
- "1": {
- "title": "content",
- "id": 22
- },
- "2": {
- "title": "id",
- "id": 23
- },
- "4": {
- "title": "objectType",
- "id": 25
- },
- "1.5": {
- "title": "geo",
- "id": 24,
- "ideas": {
- "1": {
- "title": "coordinates",
- "id": 26
- }
- },
- "attr": {
- "style": {}
- }
- }
- }
- },
- "2.9375": {
- "title": "published",
- "id": 10,
- "attr": {
- "style": {}
- },
- "ideas": {}
- },
- "2.90625": {
- "title": "provider",
- "id": 14,
- "attr": {
- "style": {}
- },
- "ideas": {
- "1": {
- "title": "id",
- "id": 20
- },
- "0.5": {
- "title": "displayName",
- "id": 21,
- "attr": {
- "style": {}
- }
- }
- }
- },
- "2.96875": {
- "title": "target",
- "id": 9,
- "attr": {
- "style": {}
- },
- "ideas": {
- "1": {
- "title": "id",
- "id": 64
- },
- "2": {
- "title": "image",
- "id": 65
- },
- "4": {
- "title": "summary",
- "id": 67
- },
- "0.5": {
- "title": "displayName",
- "id": 66,
- "attr": {
- "style": {}
- }
- },
- "0.25": {
- "title": "content",
- "id": 68,
- "attr": {
- "style": {}
- }
- }
- }
- },
- "2.984375": {
- "title": "updated",
- "id": 11,
- "attr": {
- "style": {}
- }
- },
- "2.9765625": {
- "title": "title",
- "id": 15,
- "attr": {
- "style": {}
- }
- },
- "2.9921875": {
- "title": "url",
- "id": 17,
- "attr": {
- "style": {}
- }
- },
- "1.625": {
- "title": "extensions",
- "id": 42,
- "ideas": {
- "1": {
- "title": "facebook",
- "id": 43,
- "ideas": {}
- },
- "6": {
- "title": "user_mentions",
- "id": 58,
- "ideas": {
- "1": {
- "title": "id",
- "id": 59
- },
- "2": {
- "title": "handle",
- "id": 60
- },
- "3": {
- "title": "displayName",
- "id": 61
- }
- }
- },
- "0.5": {
- "title": "hashtags",
- "id": 57,
- "attr": {
- "style": {}
- }
- },
- "0.75": {
- "title": "location",
- "id": 48,
- "ideas": {
- "1": {
- "title": "id",
- "id": 49
- },
- "2": {
- "title": "coordinates",
- "id": 50,
- "ideas": {
- "1": {
- "title": "type",
- "id": 52
- },
- "2": {
- "title": "coordinates",
- "id": 53
- }
- }
- }
- },
- "attr": {
- "style": {}
- }
- },
- "0.625": {
- "title": "likes",
- "id": 54,
- "ideas": {
- "1": {
- "title": "perspectival",
- "id": 55
- },
- "2": {
- "title": "count",
- "id": 56
- }
- },
- "attr": {
- "style": {}
- }
- },
- "0.875": {
- "title": "rebroadcasts",
- "id": 45,
- "ideas": {
- "1": {
- "title": "perspectival",
- "id": 46
- },
- "2": {
- "title": "count",
- "id": 47
- }
- },
- "attr": {
- "style": {}
- }
- }
- },
- "attr": {
- "style": {}
- }
- }
- }
- },
- "-1": {
- "title": "Facebook Post",
- "id": 3,
- "ideas": {
- "1": {
- "title": "id",
- "id": 69
- },
- "2": {
- "title": "likes",
- "id": 92,
- "ideas": {
- "1": {
- "title": "data",
- "id": 93,
- "ideas": {
- "1": {
- "title": "id",
- "id": 94
- },
- "2": {
- "title": "name",
- "id": 95
- }
- }
- }
- },
- "attr": {
- "style": {}
- }
- },
- "3": {
- "title": "message",
- "id": 74
- },
- "4": {
- "title": "picture",
- "id": 75
- },
- "7": {
- "title": "shares",
- "id": 97,
- "ideas": {
- "1": {
- "title": "count",
- "id": 98
- }
- },
- "attr": {
- "style": {}
- }
- },
- "10": {
- "title": "type",
- "id": 96
- },
- "15": {
- "title": "updated_time",
- "id": 105
- },
- "0.5": {
- "title": "actions",
- "id": 78,
- "ideas": {
- "1": {
- "title": "name",
- "id": 79
- },
- "0.5": {
- "title": "link",
- "id": 80,
- "attr": {
- "style": {}
- }
- }
- },
- "attr": {
- "style": {}
- }
- },
- "0.75": {
- "title": "application",
- "id": 100,
- "ideas": {
- "1": {
- "title": "name",
- "id": 101
- },
- "3": {
- "title": "namespace",
- "id": 103
- },
- "0.5": {
- "title": "id",
- "id": 102,
- "attr": {
- "style": {}
- }
- }
- },
- "attr": {
- "style": {}
- }
- },
- "0.875": {
- "title": "comments",
- "id": 81,
- "ideas": {
- "1": {
- "title": "data",
- "id": 82,
- "ideas": {
- "1": {
- "title": "user_likes",
- "id": 83
- },
- "0.5": {
- "title": "can_remove",
- "id": 87,
- "attr": {
- "style": {}
- }
- },
- "0.75": {
- "title": "created_time",
- "id": 91,
- "attr": {
- "style": {}
- }
- },
- "0.875": {
- "title": "id",
- "id": 85,
- "attr": {
- "style": {}
- },
- "ideas": {}
- },
- "0.8125": {
- "title": "from",
- "id": 88,
- "ideas": {
- "1": {
- "title": "id",
- "id": 89
- },
- "2": {
- "title": "name",
- "id": 90
- }
- },
- "attr": {
- "style": {}
- }
- },
- "0.9375": {
- "title": "like_count",
- "id": 86,
- "attr": {
- "style": {}
- }
- },
- "0.96875": {
- "title": "message",
- "id": 84,
- "attr": {
- "style": {}
- }
- }
- }
- }
- },
- "attr": {
- "style": {}
- }
- },
- "0.9375": {
- "title": "from",
- "id": 70,
- "ideas": {
- "1": {
- "title": "id",
- "id": 71
- },
- "2": {
- "title": "name",
- "id": 72
- },
- "0.5": {
- "title": "category",
- "id": 73,
- "attr": {
- "style": {}
- }
- }
- },
- "attr": {
- "style": {}
- }
- },
- "0.96875": {
- "title": "icon",
- "id": 77,
- "attr": {
- "style": {}
- }
- },
- "2.5": {
- "title": "link",
- "id": 76,
- "attr": {
- "style": {}
- }
- },
- "3.5": {
- "title": "object_id",
- "id": 99,
- "attr": {
- "style": {}
- }
- },
- "0.8125": {
- "title": "created_time",
- "id": 104,
- "attr": {
- "style": {}
- }
- }
- }
- }
- },
- "links": [
- {
- "ideaIdFrom": 104,
- "ideaIdTo": 10,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 21,
- "ideaIdTo": 101,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 102,
- "ideaIdTo": 20,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 43,
- "ideaIdTo": 3,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 72,
- "ideaIdTo": 32,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 13,
- "ideaIdTo": 77,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 18,
- "ideaIdTo": 76,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 74,
- "ideaIdTo": 16,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 98,
- "ideaIdTo": 47,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 105,
- "ideaIdTo": 11,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/metadata/facebook_post.png b/streams-contrib/streams-provider-facebook/metadata/facebook_post.png
deleted file mode 100644
index 2d5554f..0000000
Binary files a/streams-contrib/streams-provider-facebook/metadata/facebook_post.png and /dev/null differ
diff --git a/streams-contrib/streams-provider-facebook/pom.xml b/streams-contrib/streams-provider-facebook/pom.xml
deleted file mode 100644
index 53e851a..0000000
--- a/streams-contrib/streams-provider-facebook/pom.xml
+++ /dev/null
@@ -1,180 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>streams-contrib</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-provider-facebook</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Facebook Provider</description>
-
- <properties>
- <facebook4j.version>2.4.10</facebook4j.version>
- <skipITs>true</skipITs>
- <testDataBaseURl>http://streams.peoplepattern.com.s3.amazonaws.com/test-data/</testDataBaseURl>
- </properties>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
- <version>1.3</version>
- </dependency>
-
- <dependency>
- <groupId>org.facebook4j</groupId>
- <artifactId>facebook4j-core</artifactId>
- <version>${facebook4j.version}</version>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
-
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.facebook.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <version>1.2.1</version>
- <executions>
- <execution>
- <id>download-it-data</id>
- <phase>pre-integration-test</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
- <url>${testDataBaseURl}/${project.artifactId}.zip</url>
- <unpack>true</unpack>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- <!--<md5>df65b5642f33676313ebe4d5b69a3fff</md5>-->
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <configuration>
- <skipTests>${skipITs}</skipTests>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPageActivitySerializer.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPageActivitySerializer.java
deleted file mode 100644
index 7dd6769..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPageActivitySerializer.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.api;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.facebook.Page;
-import org.apache.streams.facebook.serializer.FacebookActivityUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang.NotImplementedException;
-
-import java.util.List;
-
-/**
- * Serializes activity posts
- * sblackmon: This class needs a rewrite
- */
-public class FacebookPageActivitySerializer implements ActivitySerializer<Page> {
-
- public static ObjectMapper mapper;
- static {
- mapper = StreamsJacksonMapper.getInstance();
- }
-
- @Override
- public String serializationFormat() {
- return "facebook_post_json_v1";
- }
-
- @Override
- public Page serialize(Activity deserialized) throws ActivitySerializerException {
- throw new NotImplementedException("Not currently supported by this deserializer");
- }
-
- @Override
- public Activity deserialize(Page page) throws ActivitySerializerException {
- Activity activity = new Activity();
-
- FacebookActivityUtil.updateActivity(page, activity);
-
- return activity;
- }
-
- @Override
- public List<Activity> deserializeAll(List<Page> serializedList) {
- throw new NotImplementedException("Not currently supported by this deserializer");
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java
deleted file mode 100644
index 0710a2c..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.api;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.facebook.Post;
-import org.apache.streams.facebook.serializer.FacebookActivityUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
-
-import java.util.List;
-
-public class FacebookPostActivitySerializer implements ActivitySerializer<org.apache.streams.facebook.Post> {
-
- public static final DateTimeFormatter FACEBOOK_FORMAT = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ssZ");
- public static final DateTimeFormatter ACTIVITY_FORMAT = ISODateTimeFormat.basicDateTime();
-
- public static final String PROVIDER_NAME = "Facebook";
-
- public static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Override
- public String serializationFormat() {
- return "facebook_post_json_v1";
- }
-
- @Override
- public Post serialize(Activity deserialized) throws ActivitySerializerException {
- throw new NotImplementedException("Not currently supported by this deserializer");
- }
-
- @Override
- public Activity deserialize(Post post) throws ActivitySerializerException {
- Activity activity = new Activity();
-
- FacebookActivityUtil.updateActivity(post, activity);
-
- return activity;
- }
-
- @Override
- public List<Activity> deserializeAll(List<Post> serializedList) {
- throw new NotImplementedException("Not currently supported by this deserializer");
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/feed/FacebookPublicFeedXmlActivitySerializer.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/feed/FacebookPublicFeedXmlActivitySerializer.java
deleted file mode 100644
index a44f982..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/feed/FacebookPublicFeedXmlActivitySerializer.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.feed;
-
-/**
- * Created with IntelliJ IDEA.
- * User: sblackmon
- * Date: 10/2/13
- * Time: 6:32 PM
- * To change this template use File | Settings | File Templates.
- */
-public class FacebookPublicFeedXmlActivitySerializer {
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/processor/FacebookTypeConverter.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/processor/FacebookTypeConverter.java
deleted file mode 100644
index a7b5c9a..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/processor/FacebookTypeConverter.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.facebook.Page;
-import org.apache.streams.facebook.Post;
-import org.apache.streams.facebook.api.FacebookPageActivitySerializer;
-import org.apache.streams.facebook.api.FacebookPostActivitySerializer;
-import org.apache.streams.facebook.provider.FacebookEventClassifier;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Queue;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * FacebookTypeConverter converts facebook data to activity streams types.
- */
-public class FacebookTypeConverter implements StreamsProcessor {
-
- public static final String STREAMS_ID = "FacebookTypeConverter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookTypeConverter.class);
-
- private ObjectMapper mapper;
-
- private Queue<StreamsDatum> inQueue;
- private Queue<StreamsDatum> outQueue;
-
- private Class inClass;
- private Class outClass;
-
- private FacebookPostActivitySerializer facebookPostActivitySerializer;
- private FacebookPageActivitySerializer facebookPageActivitySerializer;
-
- private int count = 0;
-
- public static final String TERMINATE = "TERMINATE";
-
- public FacebookTypeConverter(Class inClass, Class outClass) {
- this.inClass = inClass;
- this.outClass = outClass;
- }
-
- public Queue<StreamsDatum> getProcessorOutputQueue() {
- return outQueue;
- }
-
- public void setProcessorInputQueue(Queue<StreamsDatum> inputQueue) {
- inQueue = inputQueue;
- }
-
- /**
- * convert.
- * @param event event
- * @param inClass inClass
- * @param outClass outClass
- * @return Object
- * @throws ActivitySerializerException ActivitySerializerException
- * @throws JsonProcessingException JsonProcessingException
- */
- public Object convert(ObjectNode event, Class inClass, Class outClass) throws ActivitySerializerException, JsonProcessingException {
-
- Object result = null;
-
- if ( outClass.equals( Activity.class )) {
- LOGGER.debug("ACTIVITY");
- if (inClass.equals(Post.class)) {
- LOGGER.debug("POST");
- result = facebookPostActivitySerializer.deserialize(mapper.convertValue(event, Post.class));
- } else if (inClass.equals(Page.class)) {
- LOGGER.debug("PAGE");
- result = facebookPageActivitySerializer.deserialize(mapper.convertValue(event, Page.class));
- }
- } else if ( outClass.equals( Post.class )) {
- LOGGER.debug("POST");
- result = mapper.convertValue(event, Post.class);
- } else if ( outClass.equals(Page.class)) {
- LOGGER.debug("PAGE");
- result = mapper.convertValue(event, Page.class);
- } else if ( outClass.equals( ObjectNode.class )) {
- LOGGER.debug("OBJECTNODE");
- result = mapper.convertValue(event, ObjectNode.class);
- }
-
- // no supported conversion were applied
- if ( result != null ) {
- count ++;
- return result;
- }
-
- LOGGER.debug("CONVERT FAILED");
-
- return null;
- }
-
- // TODO: use standard validation
- public boolean validate(Object document, Class klass) {
- return true;
- }
-
- // TODO: replace with standard validation
- public boolean isValidJSON(final String json) {
- boolean valid = false;
- try {
- final JsonParser parser = new ObjectMapper().getJsonFactory()
- .createJsonParser(json);
- while (parser.nextToken() != null) {
- }
- valid = true;
- } catch (IOException ioe) {
- LOGGER.warn("validate: {}", ioe);
- }
-
- return valid;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- StreamsDatum result = null;
-
- try {
- Object item = entry.getDocument();
- ObjectNode node;
-
- LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
-
- if ( item instanceof String ) {
-
- // if the target is string, just pass-through
- if ( String.class.equals(outClass)) {
- result = entry;
- } else {
- // first check for valid json
- node = (ObjectNode)mapper.readTree((String)item);
-
- // since data is coming from outside provider, we don't know what type the events are
- // for now we'll assume post
- Class inClass = FacebookEventClassifier.detectClass((String) item);
-
- Object out = convert(node, inClass, outClass);
-
- if ( out != null && validate(out, outClass)) {
- result = new StreamsDatum(out);
- }
- }
-
- } else if ( item instanceof ObjectNode) {
-
- // first check for valid json
- node = mapper.valueToTree(item);
-
- Class inClass = FacebookEventClassifier.detectClass(mapper.writeValueAsString(item));
-
- Object out = convert(node, inClass, outClass);
-
- if ( out != null && validate(out, outClass)) {
- result = new StreamsDatum(out);
- }
- } else if (item instanceof Post || item instanceof Page) {
- Object out = convert(mapper.convertValue(item, ObjectNode.class), inClass, outClass);
-
- if ( out != null && validate(out, outClass)) {
- result = new StreamsDatum(out);
- }
- }
- } catch (Exception ex) {
- LOGGER.error("Exception switching types : {}", ex);
- if (ex instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- }
-
- if ( result != null ) {
- return Stream.of(result).collect(Collectors.toList());
- } else {
- return new ArrayList<>();
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
- mapper = StreamsJacksonMapper.getInstance();
-
- facebookPageActivitySerializer = new FacebookPageActivitySerializer();
- facebookPostActivitySerializer = new FacebookPostActivitySerializer();
- }
-
- @Override
- public void cleanUp() {}
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookDataCollector.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookDataCollector.java
deleted file mode 100644
index 0a13b64..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookDataCollector.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.facebook.FacebookConfiguration;
-import org.apache.streams.facebook.IdConfig;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
-import org.apache.streams.util.oauth.tokens.tokenmanager.SimpleTokenManager;
-import org.apache.streams.util.oauth.tokens.tokenmanager.impl.BasicTokenManager;
-
-import com.google.common.annotations.VisibleForTesting;
-import facebook4j.Facebook;
-import facebook4j.FacebookFactory;
-import facebook4j.conf.ConfigurationBuilder;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Abstract data collector for Facebook. Iterates over ids and queues data to be output
- * by a {@link org.apache.streams.core.StreamsProvider}
- */
-public abstract class FacebookDataCollector implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookDataCollector.class);
- private static final String READ_ONLY = "read_streams";
-
- @VisibleForTesting
- protected AtomicBoolean isComplete;
- protected BackOffStrategy backOff;
-
- private FacebookConfiguration config;
- private BlockingQueue<StreamsDatum> queue;
- private SimpleTokenManager<String> authTokens;
-
- /**
- * FacebookDataCollector constructor.
- * @param config config
- * @param queue queue
- */
- public FacebookDataCollector(FacebookConfiguration config, BlockingQueue<StreamsDatum> queue) {
- this.config = config;
- this.queue = queue;
- this.isComplete = new AtomicBoolean(false);
- this.backOff = new ExponentialBackOffStrategy(5);
- this.authTokens = new BasicTokenManager<>();
- if (config.getUserAccessTokens() != null) {
- for (String token : config.getUserAccessTokens()) {
- this.authTokens.addTokenToPool(token);
- }
- }
- }
-
- /**
- * Returns true when the collector has finished querying facebook and has queued all data
- * for the provider.
- * @return isComplete
- */
- public boolean isComplete() {
- return this.isComplete.get();
- }
-
- /**
- * Queues facebook data.
- * @param data data
- * @param id id
- */
- protected void outputData(Object data, String id) {
- try {
- this.queue.put(new StreamsDatum(data, id));
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * Gets a Facebook client. If multiple authenticated users for this app are available
- * it will rotate through the users oauth credentials
- * @return client
- */
- protected Facebook getNextFacebookClient() {
- ConfigurationBuilder cb = new ConfigurationBuilder();
- cb.setDebugEnabled(true);
- cb.setOAuthPermissions(READ_ONLY);
- cb.setOAuthAppId(this.config.getOauth().getAppId());
- cb.setOAuthAppSecret(this.config.getOauth().getAppSecret());
- if (this.authTokens.numAvailableTokens() > 0) {
- cb.setOAuthAccessToken(this.authTokens.getNextAvailableToken());
- } else {
- cb.setOAuthAccessToken(this.config.getOauth().getAppAccessToken());
- LOGGER.debug("appAccessToken : {}", this.config.getOauth().getAppAccessToken());
- }
- cb.setJSONStoreEnabled(true);
- if (StringUtils.isNotEmpty(config.getVersion())) {
- cb.setRestBaseURL("https://graph.facebook.com/" + config.getVersion() + "/");
- }
- LOGGER.debug("appId : {}", this.config.getOauth().getAppId());
- LOGGER.debug("appSecret: {}", this.config.getOauth().getAppSecret());
- FacebookFactory ff = new FacebookFactory(cb.build());
- return ff.getInstance();
- }
-
- /**
- * Queries facebook and queues the resulting data.
- * @param id id
- * @throws Exception Exception
- */
- protected abstract void getData(IdConfig id) throws Exception;
-
-
- @Override
- public void run() {
- for ( IdConfig id : this.config.getIds()) {
- try {
- getData(id);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- } catch (Exception ex) {
- LOGGER.error("Caught Exception while trying to poll data for page : {}", id);
- LOGGER.error("Exception while getting page feed data: {}", ex);
- }
- }
- this.isComplete.set(true);
- }
-
- @VisibleForTesting
- protected BlockingQueue<StreamsDatum> getQueue() {
- return queue;
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookEventClassifier.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookEventClassifier.java
deleted file mode 100644
index 17dae1d..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookEventClassifier.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.provider;
-
-import org.apache.streams.facebook.Page;
-import org.apache.streams.facebook.Post;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Objects;
-
-/**
- * FacebookEventClassifier classifies facebook events.
- */
-public class FacebookEventClassifier {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookEventClassifier.class);
-
- /**
- * detectClass from json string.
- * @param json json string
- * @return detected Class
- */
- public static Class detectClass( String json ) {
-
- Objects.requireNonNull(json);
- Preconditions.checkArgument(StringUtils.isNotEmpty(json));
-
- ObjectNode objectNode;
- try {
- objectNode = (ObjectNode) StreamsJacksonMapper.getInstance().readTree(json);
- } catch (IOException ex) {
- LOGGER.error("Exception while trying to detect class: {}", ex.getMessage());
- return null;
- }
-
- if ( objectNode.findValue("about") != null) {
- return Page.class;
- } else if ( objectNode.findValue("statusType") != null ) {
- return Post.class;
- } else {
- return Post.class;
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookFriendFeedProvider.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookFriendFeedProvider.java
deleted file mode 100644
index 7f0a34d..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookFriendFeedProvider.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.provider;
-
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.facebook.FacebookUserInformationConfiguration;
-import org.apache.streams.facebook.FacebookUserstreamConfiguration;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigRenderOptions;
-import facebook4j.Facebook;
-import facebook4j.FacebookException;
-import facebook4j.FacebookFactory;
-import facebook4j.Friend;
-import facebook4j.Paging;
-import facebook4j.Post;
-import facebook4j.ResponseList;
-import facebook4j.conf.ConfigurationBuilder;
-import facebook4j.json.DataObjectFactory;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.Iterator;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-public class FacebookFriendFeedProvider implements StreamsProvider, Serializable {
-
- private static final String STREAMS_ID = "FacebookFriendFeedProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookFriendFeedProvider.class);
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private static final String ALL_PERMISSIONS = "ads_management,ads_read,create_event,create_note,email,export_stream,friends_about_me,friends_actions.books,friends_actions.music,friends_actions.news,friends_actions.video,friends_activities,friends_birthday,friends_education_history,friends_events,friends_games_activity,friends_groups,friends_hometown,friends_interests,friends_likes,friends_location,friends_notes,friends_online_presence,friends_photo_video_tags,friends_photos,friends_que [...]
- private FacebookUserstreamConfiguration configuration;
-
- private Class klass;
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- protected volatile Queue<StreamsDatum> providerQueue = new LinkedBlockingQueue<>();
-
- public FacebookUserstreamConfiguration getConfig() {
- return configuration;
- }
-
- public void setConfig(FacebookUserstreamConfiguration config) {
- this.configuration = config;
- }
-
- protected Iterator<String[]> idsBatches;
-
- protected ExecutorService executor;
-
- protected DateTime start;
- protected DateTime end;
-
- protected final AtomicBoolean running = new AtomicBoolean();
-
- private DatumStatusCounter countersCurrent = new DatumStatusCounter();
- private DatumStatusCounter countersTotal = new DatumStatusCounter();
-
- private static ExecutorService newFixedThreadPoolWithQueueSize(int numThreads, int queueSize) {
- return new ThreadPoolExecutor(numThreads, numThreads,
- 5000L, TimeUnit.MILLISECONDS,
- new ArrayBlockingQueue<>(queueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
- }
-
- /**
- * FacebookFriendFeedProvider constructor - resolves FacebookUserInformationConfiguration from JVM 'facebook'.
- */
- public FacebookFriendFeedProvider() {
- Config config = StreamsConfigurator.getConfig().getConfig("facebook");
- FacebookUserInformationConfiguration configuration;
- try {
- configuration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), FacebookUserInformationConfiguration.class);
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- /**
- * FacebookFriendFeedProvider constructor - uses supplied FacebookUserInformationConfiguration.
- */
- public FacebookFriendFeedProvider(FacebookUserstreamConfiguration config) {
- this.configuration = config;
- }
-
- /**
- * FacebookFriendFeedProvider constructor - output supplied Class.
- * @param klass Class
- */
- public FacebookFriendFeedProvider(Class klass) {
- Config config = StreamsConfigurator.getConfig().getConfig("facebook");
- FacebookUserInformationConfiguration configuration;
- try {
- configuration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), FacebookUserInformationConfiguration.class);
- } catch (IOException ex) {
- ex.printStackTrace();
- return;
- }
- this.klass = klass;
- }
-
- public FacebookFriendFeedProvider(FacebookUserstreamConfiguration config, Class klass) {
- this.configuration = config;
- this.klass = klass;
- }
-
- public Queue<StreamsDatum> getProviderQueue() {
- return this.providerQueue;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
- shutdownAndAwaitTermination(executor);
- running.set(true);
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- synchronized (FacebookUserstreamProvider.class) {
- current = new StreamsResultSet(new ConcurrentLinkedQueue<>(providerQueue));
- current.setCounter(new DatumStatusCounter());
- current.getCounter().add(countersCurrent);
- countersTotal.add(countersCurrent);
- countersCurrent = new DatumStatusCounter();
- providerQueue.clear();
- }
-
- return current;
-
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- LOGGER.debug("{} readNew", STREAMS_ID);
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- LOGGER.debug("{} readRange", STREAMS_ID);
- this.start = start;
- this.end = end;
- readCurrent();
- return (StreamsResultSet)providerQueue.iterator();
- }
-
- @Override
- public boolean isRunning() {
- return running.get();
- }
-
- void shutdownAndAwaitTermination(ExecutorService pool) {
- pool.shutdown(); // Disable new tasks from being submitted
- try {
- // Wait a while for existing tasks to terminate
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- pool.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- System.err.println("Pool did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- pool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- executor = newFixedThreadPoolWithQueueSize(5, 20);
-
- Objects.requireNonNull(providerQueue);
- Objects.requireNonNull(this.klass);
- Objects.requireNonNull(configuration.getOauth().getAppId());
- Objects.requireNonNull(configuration.getOauth().getAppSecret());
- Objects.requireNonNull(configuration.getOauth().getUserAccessToken());
-
- Facebook client = getFacebookClient();
-
- try {
- ResponseList<Friend> friendResponseList = client.friends().getFriends();
- Paging<Friend> friendPaging;
- do {
-
- for ( Friend friend : friendResponseList ) {
- executor.submit(new FacebookFriendFeedTask(this, friend.getId()));
- }
- friendPaging = friendResponseList.getPaging();
- friendResponseList = client.fetchNext(friendPaging);
- }
- while ( friendPaging != null
- &&
- friendResponseList != null );
- } catch (FacebookException ex) {
- ex.printStackTrace();
- }
-
- }
-
- protected Facebook getFacebookClient() {
-
- ConfigurationBuilder cb = new ConfigurationBuilder();
- cb.setDebugEnabled(true)
- .setOAuthAppId(configuration.getOauth().getAppId())
- .setOAuthAppSecret(configuration.getOauth().getAppSecret())
- .setOAuthAccessToken(configuration.getOauth().getUserAccessToken())
- .setOAuthPermissions(ALL_PERMISSIONS)
- .setJSONStoreEnabled(true)
- .setClientVersion("v1.0");
-
- FacebookFactory ff = new FacebookFactory(cb.build());
-
- return ff.getInstance();
- }
-
- @Override
- public void cleanUp() {
- shutdownAndAwaitTermination(executor);
- }
-
- private class FacebookFriendFeedTask implements Runnable {
-
- FacebookFriendFeedProvider provider;
- Facebook client;
- String id;
-
- public FacebookFriendFeedTask(FacebookFriendFeedProvider provider, String id) {
- this.provider = provider;
- this.id = id;
- }
-
- int last_count = 0;
- int page_count = 0;
- int item_count = 0;
-
- @Override
- public void run() {
- client = provider.getFacebookClient();
- try {
- ResponseList<Post> postResponseList = client.getFeed(id);
- Paging<Post> postPaging;
- do {
- last_count = postResponseList.getCount();
- for (Post item : postResponseList) {
- String json = DataObjectFactory.getRawJSON(item);
- org.apache.streams.facebook.Post post = mapper.readValue(json, org.apache.streams.facebook.Post.class);
- try {
- lock.readLock().lock();
- ComponentUtils.offerUntilSuccess(new StreamsDatum(post), providerQueue);
- item_count++;
- } finally {
- lock.readLock().unlock();
- }
- }
- page_count++;
- postPaging = postResponseList.getPaging();
- postResponseList = client.fetchNext(postPaging);
- }
- while ( shouldContinuePulling(postPaging, postResponseList) );
-
- LOGGER.info("item_count: {} last_count: {} page_count: {} ", item_count, last_count, page_count);
-
- } catch (Exception ex) {
- ex.printStackTrace();
- }
-
- }
-
- public boolean shouldContinuePulling(Paging<Post> postPaging, ResponseList<Post> postResponseList) {
- return (
- postPaging != null
- &&
- postResponseList != null
- );
- }
- }
-
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookFriendUpdatesProvider.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookFriendUpdatesProvider.java
deleted file mode 100644
index 0aface9..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookFriendUpdatesProvider.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.provider;
-
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.facebook.FacebookUserInformationConfiguration;
-import org.apache.streams.facebook.FacebookUserstreamConfiguration;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigRenderOptions;
-import facebook4j.Facebook;
-import facebook4j.FacebookException;
-import facebook4j.FacebookFactory;
-import facebook4j.Friend;
-import facebook4j.Paging;
-import facebook4j.Post;
-import facebook4j.ResponseList;
-import facebook4j.conf.ConfigurationBuilder;
-import facebook4j.json.DataObjectFactory;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-/**
- * FacebookFriendUpdatesProvider provides updates from friend feed.
- */
-public class FacebookFriendUpdatesProvider implements StreamsProvider, Serializable {
-
- private static final String STREAMS_ID = "FacebookFriendPostsProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookFriendUpdatesProvider.class);
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private static final String ALL_PERMISSIONS =
- "ads_management,ads_read,create_event,create_note,email,export_stream,friends_about_me,friends_actions.books,friends_actions.music,friends_actions.news,friends_actions.video,friends_activities,friends_birthday,friends_education_history,friends_events,friends_games_activity,friends_groups,friends_hometown,friends_interests,friends_likes,friends_location,friends_notes,friends_online_presence,friends_photo_video_tags,friends_photos,friends_questions,friends_relationship_details,friend [...]
-
- private FacebookUserstreamConfiguration configuration;
-
- private Class klass;
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- protected volatile Queue<StreamsDatum> providerQueue = new LinkedBlockingQueue<>();
-
- public FacebookUserstreamConfiguration getConfig() {
- return configuration;
- }
-
- public void setConfig(FacebookUserstreamConfiguration config) {
- this.configuration = config;
- }
-
- protected Iterator<String[]> idsBatches;
-
- protected ExecutorService executor;
-
- protected DateTime start;
- protected DateTime end;
-
- protected final AtomicBoolean running = new AtomicBoolean();
-
- private DatumStatusCounter countersCurrent = new DatumStatusCounter();
- private DatumStatusCounter countersTotal = new DatumStatusCounter();
-
- // TODO: factor this out.
- private static ExecutorService newFixedThreadPoolWithQueueSize(int numThreads, int queueSize) {
- return new ThreadPoolExecutor(numThreads, numThreads,
- 5000L, TimeUnit.MILLISECONDS,
- new ArrayBlockingQueue<>(queueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
- }
-
- /**
- * FacebookFriendUpdatesProvider constructor - resolves FacebookUserInformationConfiguration from JVM 'facebook'.
- */
- public FacebookFriendUpdatesProvider() {
- Config config = StreamsConfigurator.getConfig().getConfig("facebook");
- FacebookUserInformationConfiguration configuration;
- try {
- configuration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), FacebookUserInformationConfiguration.class);
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- /**
- * FacebookFriendUpdatesProvider constructor - uses supplied FacebookUserstreamConfiguration.
- */
- public FacebookFriendUpdatesProvider(FacebookUserstreamConfiguration config) {
- this.configuration = config;
- }
-
- /**
- * FacebookFriendUpdatesProvider constructor.
- * uses supplied output Class.
- * resolves FacebookUserInformationConfiguration from JVM 'facebook.
- */
- public FacebookFriendUpdatesProvider(Class klass) {
- Config config = StreamsConfigurator.getConfig().getConfig("facebook");
- FacebookUserInformationConfiguration configuration;
- try {
- configuration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), FacebookUserInformationConfiguration.class);
- } catch (IOException ex) {
- ex.printStackTrace();
- return;
- }
- this.klass = klass;
- }
-
- /**
- * FacebookFriendUpdatesProvider constructor.
- * uses supplied FacebookUserstreamConfiguration.
- * uses supplied output Class.
- */
- public FacebookFriendUpdatesProvider(FacebookUserstreamConfiguration config, Class klass) {
- this.configuration = config;
- this.klass = klass;
- }
-
- public Queue<StreamsDatum> getProviderQueue() {
- return this.providerQueue;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
- running.set(true);
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- Preconditions.checkArgument(idsBatches.hasNext());
-
- LOGGER.info("readCurrent");
-
- // return stuff
-
- LOGGER.info("Finished. Cleaning up...");
-
- LOGGER.info("Providing {} docs", providerQueue.size());
-
- StreamsResultSet result = new StreamsResultSet(providerQueue);
- running.set(false);
-
- LOGGER.info("Exiting");
-
- return result;
-
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- LOGGER.debug("{} readNew", STREAMS_ID);
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- LOGGER.debug("{} readRange", STREAMS_ID);
- this.start = start;
- this.end = end;
- readCurrent();
- return (StreamsResultSet)providerQueue.iterator();
- }
-
- @Override
- public boolean isRunning() {
- return running.get();
- }
-
- void shutdownAndAwaitTermination(ExecutorService pool) {
- pool.shutdown(); // Disable new tasks from being submitted
- try {
- // Wait a while for existing tasks to terminate
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- pool.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- System.err.println("Pool did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- pool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- executor = newFixedThreadPoolWithQueueSize(5, 20);
-
- Objects.requireNonNull(providerQueue);
- Objects.requireNonNull(this.klass);
- Objects.requireNonNull(configuration.getOauth().getAppId());
- Objects.requireNonNull(configuration.getOauth().getAppSecret());
- Objects.requireNonNull(configuration.getOauth().getUserAccessToken());
-
- Facebook client = getFacebookClient();
-
- try {
- ResponseList<Friend> friendResponseList = client.friends().getFriends();
- Paging<Friend> friendPaging;
- do {
- friendPaging = friendResponseList.getPaging();
- friendResponseList = client.fetchNext(friendPaging);
- }
- while ( friendPaging != null
- &&
- friendResponseList != null );
- } catch (FacebookException ex) {
- ex.printStackTrace();
- }
-
- }
-
- protected Facebook getFacebookClient() {
-
- ConfigurationBuilder cb = new ConfigurationBuilder();
- cb.setDebugEnabled(true)
- .setOAuthAppId(configuration.getOauth().getAppId())
- .setOAuthAppSecret(configuration.getOauth().getAppSecret())
- .setOAuthAccessToken(configuration.getOauth().getUserAccessToken())
- .setOAuthPermissions(ALL_PERMISSIONS)
- .setJSONStoreEnabled(true)
- .setClientVersion("v1.0");
-
- FacebookFactory ff = new FacebookFactory(cb.build());
-
- return ff.getInstance();
- }
-
- @Override
- public void cleanUp() {
- shutdownAndAwaitTermination(executor);
- }
-
- private class FacebookFeedPollingTask implements Runnable {
-
- FacebookUserstreamProvider provider;
- Facebook client;
-
- private Set<Post> priorPollResult = new HashSet<>();
-
- public FacebookFeedPollingTask(FacebookUserstreamProvider facebookUserstreamProvider) {
- provider = facebookUserstreamProvider;
- }
-
- @Override
- public void run() {
- client = provider.getFacebookClient();
- while (provider.isRunning()) {
- try {
- ResponseList<Post> postResponseList = client.getHome();
- Set<Post> update = new HashSet<>(postResponseList);
- Set<Post> repeats = priorPollResult.stream().filter(update::contains).collect(Collectors.toSet());
- Set<Post> entrySet = update.stream().filter((x) -> !repeats.contains(x)).collect(Collectors.toSet());
- for (Post item : entrySet) {
- String json = DataObjectFactory.getRawJSON(item);
- org.apache.streams.facebook.Post post = mapper.readValue(json, org.apache.streams.facebook.Post.class);
- try {
- lock.readLock().lock();
- ComponentUtils.offerUntilSuccess(new StreamsDatum(post), providerQueue);
- countersCurrent.incrementAttempt();
- } finally {
- lock.readLock().unlock();
- }
- }
- priorPollResult = update;
- Thread.sleep(configuration.getPollIntervalMillis());
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
- }
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookProvider.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookProvider.java
deleted file mode 100644
index a17abb1..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookProvider.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.provider;
-
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.facebook.FacebookConfiguration;
-import org.apache.streams.facebook.IdConfig;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.ComponentUtils;
-import org.apache.streams.util.SerializationUtil;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.typesafe.config.ConfigRenderOptions;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Abstract {@link org.apache.streams.core.StreamsProvider} for facebook.
- */
-public abstract class FacebookProvider implements StreamsProvider {
-
- private static final String STREAMS_ID = "FacebookProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookProvider.class);
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
- private static final int MAX_BATCH_SIZE = 2000;
-
- protected FacebookConfiguration configuration;
- protected BlockingQueue<StreamsDatum> datums;
-
- private AtomicBoolean isComplete;
- private ListeningExecutorService executor;
- List<ListenableFuture<Object>> futures = new ArrayList<>();
-
- private FacebookDataCollector dataCollector;
-
- /**
- * FacebookProvider constructor - resolves FacebookConfiguration from JVM 'facebook'.
- */
- public FacebookProvider() {
- try {
- this.configuration = MAPPER.readValue(StreamsConfigurator.getConfig().getConfig("facebook").root().render(ConfigRenderOptions.concise()), FacebookConfiguration.class);
- } catch (IOException ioe) {
- LOGGER.error("Exception trying to read default config : {}", ioe);
- }
- }
-
- /**
- * FacebookProvider constructor - uses supplied FacebookConfiguration.
- */
- public FacebookProvider(FacebookConfiguration configuration) {
- this.configuration = SerializationUtil.cloneBySerialization(configuration);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
- ListenableFuture future = executor.submit(getDataCollector());
- futures.add(future);
- executor.shutdown();
- }
-
- protected abstract FacebookDataCollector getDataCollector();
-
- @Override
- public StreamsResultSet readCurrent() {
- int batchSize = 0;
- BlockingQueue<StreamsDatum> batch = new LinkedBlockingQueue<>();
- while (!this.datums.isEmpty() && batchSize < MAX_BATCH_SIZE) {
- ComponentUtils.offerUntilSuccess(ComponentUtils.pollWhileNotEmpty(this.datums), batch);
- ++batchSize;
- }
- return new StreamsResultSet(batch);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.datums = new LinkedBlockingQueue<>();
- this.isComplete = new AtomicBoolean(false);
- this.executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
- }
-
- @Override
- public void cleanUp() {
- ComponentUtils.shutdownExecutor(executor, 5, 5);
- executor = null;
- }
-
- /**
- * Overrides the ids and addedAfter time in the configuration.
- * @param idsToAfterDate idsToAfterDate
- */
- public void overrideIds(Map<String, DateTime> idsToAfterDate) {
- Set<IdConfig> ids = new HashSet<>();
- for (String id : idsToAfterDate.keySet()) {
- IdConfig idConfig = new IdConfig();
- idConfig.setId(id);
- idConfig.setAfterDate(idsToAfterDate.get(id));
- ids.add(idConfig);
- }
- this.configuration.setIds(ids);
- }
-
- @Override
- public boolean isRunning() {
- if (datums.isEmpty() && executor.isTerminated() && Futures.allAsList(futures).isDone()) {
- LOGGER.info("Completed");
- isComplete.set(true);
- LOGGER.info("Exiting");
- }
- return !isComplete.get();
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookUserInformationProvider.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookUserInformationProvider.java
deleted file mode 100644
index fdf6867..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookUserInformationProvider.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.provider;
-
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.facebook.FacebookUserInformationConfiguration;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigRenderOptions;
-import facebook4j.Facebook;
-import facebook4j.FacebookException;
-import facebook4j.FacebookFactory;
-import facebook4j.Friend;
-import facebook4j.Paging;
-import facebook4j.ResponseList;
-import facebook4j.User;
-import facebook4j.conf.ConfigurationBuilder;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-
-public class FacebookUserInformationProvider implements StreamsProvider, Serializable {
-
- public static final String STREAMS_ID = "FacebookUserInformationProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookUserInformationProvider.class);
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private static final String ALL_PERMISSIONS = "ads_management,ads_read,create_event,create_note,email,export_stream,friends_about_me,friends_actions.books,friends_actions.music,friends_actions.news,friends_actions.video,friends_activities,friends_birthday,friends_education_history,friends_events,friends_games_activity,friends_groups,friends_hometown,friends_interests,friends_likes,friends_location,friends_notes,friends_online_presence,friends_photo_video_tags,friends_photos,friends_que [...]
- private FacebookUserInformationConfiguration facebookUserInformationConfiguration;
-
- private Class klass;
- protected volatile Queue<StreamsDatum> providerQueue = new LinkedBlockingQueue<>();
-
- public FacebookUserInformationConfiguration getConfig() {
- return facebookUserInformationConfiguration;
- }
-
- public void setConfig(FacebookUserInformationConfiguration config) {
- this.facebookUserInformationConfiguration = config;
- }
-
- protected Iterator<String[]> idsBatches;
-
- protected ExecutorService executor;
-
- protected DateTime start;
- protected DateTime end;
-
- protected final AtomicBoolean running = new AtomicBoolean();
-
- private static ExecutorService newFixedThreadPoolWithQueueSize(int numThreads, int queueSize) {
- return new ThreadPoolExecutor(numThreads, numThreads,
- 5000L, TimeUnit.MILLISECONDS,
- new ArrayBlockingQueue<>(queueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
- }
-
- /**
- * FacebookUserInformationProvider constructor - resolves FacebookUserInformationConfiguration from JVM 'facebook'.
- */
- public FacebookUserInformationProvider() {
- Config config = StreamsConfigurator.getConfig().getConfig("facebook");
- FacebookUserInformationConfiguration facebookUserInformationConfiguration;
- try {
- facebookUserInformationConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), FacebookUserInformationConfiguration.class);
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- /**
- * FacebookUserInformationProvider constructor - uses supplie FacebookUserInformationConfiguration.
- * @param config
- */
- public FacebookUserInformationProvider(FacebookUserInformationConfiguration config) {
- this.facebookUserInformationConfiguration = config;
- }
-
- public FacebookUserInformationProvider(Class klass) {
- Config config = StreamsConfigurator.getConfig().getConfig("facebook");
- FacebookUserInformationConfiguration facebookUserInformationConfiguration;
- try {
- facebookUserInformationConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), FacebookUserInformationConfiguration.class);
- } catch (IOException ex) {
- ex.printStackTrace();
- return;
- }
- this.klass = klass;
- }
-
- public FacebookUserInformationProvider(FacebookUserInformationConfiguration config, Class klass) {
- this.facebookUserInformationConfiguration = config;
- this.klass = klass;
- }
-
- public Queue<StreamsDatum> getProviderQueue() {
- return this.providerQueue;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
- running.set(true);
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- Preconditions.checkArgument(idsBatches.hasNext());
-
- LOGGER.info("readCurrent");
-
- Facebook client = getFacebookClient();
-
- try {
- User me = client.users().getMe();
- String json = mapper.writeValueAsString(me);
- providerQueue.add(
- new StreamsDatum(json, DateTime.now())
- );
- } catch (JsonProcessingException | FacebookException ex) {
- ex.printStackTrace();
- }
-
- if (idsBatches.hasNext()) {
- while (idsBatches.hasNext()) {
- try {
- List<User> userList = client.users().getUsers(idsBatches.next());
- for (User user : userList) {
-
- try {
- String json = mapper.writeValueAsString(user);
- providerQueue.add(
- new StreamsDatum(json, DateTime.now())
- );
- } catch (JsonProcessingException ex) {
- LOGGER.trace("JsonProcessingException", ex);
- }
- }
-
- } catch (FacebookException ex) {
- ex.printStackTrace();
- }
- }
- } else {
- try {
- ResponseList<Friend> friendResponseList = client.friends().getFriends();
- Paging<Friend> friendPaging;
- do {
-
- for ( Friend friend : friendResponseList ) {
-
- String json;
- try {
- json = mapper.writeValueAsString(friend);
- providerQueue.add(
- new StreamsDatum(json)
- );
- } catch (JsonProcessingException ex) {
- LOGGER.trace("JsonProcessingException", ex);
- }
- }
- friendPaging = friendResponseList.getPaging();
- friendResponseList = client.fetchNext(friendPaging);
- }
- while ( friendPaging != null
- &&
- friendResponseList != null );
- } catch (FacebookException ex) {
- ex.printStackTrace();
- }
-
- }
-
- LOGGER.info("Finished. Cleaning up...");
-
- LOGGER.info("Providing {} docs", providerQueue.size());
-
- StreamsResultSet result = new StreamsResultSet(providerQueue);
- running.set(false);
-
- LOGGER.info("Exiting");
-
- return result;
-
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- LOGGER.debug("{} readNew", STREAMS_ID);
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- LOGGER.debug("{} readRange", STREAMS_ID);
- this.start = start;
- this.end = end;
- readCurrent();
- return (StreamsResultSet)providerQueue.iterator();
- }
-
- @Override
- public boolean isRunning() {
- return running.get();
- }
-
- void shutdownAndAwaitTermination(ExecutorService pool) {
- pool.shutdown(); // Disable new tasks from being submitted
- try {
- // Wait a while for existing tasks to terminate
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- pool.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- System.err.println("Pool did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- pool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- executor = MoreExecutors.listeningDecorator(newFixedThreadPoolWithQueueSize(5, 20));
-
- Objects.requireNonNull(providerQueue);
- Objects.requireNonNull(this.klass);
- Objects.requireNonNull(facebookUserInformationConfiguration.getOauth().getAppId());
- Objects.requireNonNull(facebookUserInformationConfiguration.getOauth().getAppSecret());
- Objects.requireNonNull(facebookUserInformationConfiguration.getOauth().getUserAccessToken());
- Objects.requireNonNull(facebookUserInformationConfiguration.getInfo());
-
- List<String> ids = new ArrayList<>();
- List<String[]> idsBatches = new ArrayList<>();
-
- for (String s : facebookUserInformationConfiguration.getInfo()) {
- if (s != null) {
- ids.add(s);
-
- if (ids.size() >= 100) {
- // add the batch
- idsBatches.add(ids.toArray(new String[ids.size()]));
- // reset the Ids
- ids = new ArrayList<>();
- }
-
- }
- }
-
- if (ids.size() > 0) {
- idsBatches.add(ids.toArray(new String[ids.size()]));
- }
-
- this.idsBatches = idsBatches.iterator();
- }
-
- protected Facebook getFacebookClient() {
- ConfigurationBuilder cb = new ConfigurationBuilder();
- cb.setDebugEnabled(true)
- .setOAuthAppId(facebookUserInformationConfiguration.getOauth().getAppId())
- .setOAuthAppSecret(facebookUserInformationConfiguration.getOauth().getAppSecret())
- .setOAuthAccessToken(facebookUserInformationConfiguration.getOauth().getUserAccessToken())
- .setOAuthPermissions(ALL_PERMISSIONS)
- .setJSONStoreEnabled(true)
- .setClientVersion("v1.0");
-
- FacebookFactory ff = new FacebookFactory(cb.build());
-
- return ff.getInstance();
- }
-
- @Override
- public void cleanUp() {
- shutdownAndAwaitTermination(executor);
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookUserstreamProvider.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookUserstreamProvider.java
deleted file mode 100644
index d875342..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/FacebookUserstreamProvider.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.provider;
-
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.facebook.FacebookUserInformationConfiguration;
-import org.apache.streams.facebook.FacebookUserstreamConfiguration;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigRenderOptions;
-import facebook4j.Facebook;
-import facebook4j.FacebookException;
-import facebook4j.FacebookFactory;
-import facebook4j.Post;
-import facebook4j.ResponseList;
-import facebook4j.conf.ConfigurationBuilder;
-import facebook4j.json.DataObjectFactory;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-public class FacebookUserstreamProvider implements StreamsProvider, Serializable {
-
- private static final String STREAMS_ID = "FacebookUserstreamProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookUserstreamProvider.class);
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private static final String ALL_PERMISSIONS = "read_stream";
- private FacebookUserstreamConfiguration configuration;
-
- private Class klass;
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- protected volatile Queue<StreamsDatum> providerQueue = new LinkedBlockingQueue<>();
-
- public FacebookUserstreamConfiguration getConfig() {
- return configuration;
- }
-
- public void setConfig(FacebookUserstreamConfiguration config) {
- this.configuration = config;
- }
-
- protected ExecutorService executor;
-
- protected DateTime start;
- protected DateTime end;
-
- protected final AtomicBoolean running = new AtomicBoolean();
-
- private DatumStatusCounter countersCurrent = new DatumStatusCounter();
- private DatumStatusCounter countersTotal = new DatumStatusCounter();
-
- protected Facebook client;
-
- private static ExecutorService newFixedThreadPoolWithQueueSize(int numThreads, int queueSize) {
- return new ThreadPoolExecutor(numThreads, numThreads,
- 5000L, TimeUnit.MILLISECONDS,
- new ArrayBlockingQueue<>(queueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
- }
-
- /**
- * FacebookUserstreamProvider constructor.
- */
- public FacebookUserstreamProvider() {
- Config config = StreamsConfigurator.getConfig().getConfig("facebook");
- FacebookUserInformationConfiguration facebookUserInformationConfiguration;
- try {
- facebookUserInformationConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), FacebookUserInformationConfiguration.class);
- } catch (IOException ex) {
- ex.printStackTrace();
- }
- }
-
- /**
- * FacebookUserstreamProvider constructor.
- * @param config config
- */
- public FacebookUserstreamProvider(FacebookUserstreamConfiguration config) {
- this.configuration = config;
- }
-
- /**
- * FacebookUserstreamProvider constructor.
- * @param klass output Class
- */
- public FacebookUserstreamProvider(Class klass) {
- Config config = StreamsConfigurator.getConfig().getConfig("facebook");
- FacebookUserInformationConfiguration facebookUserInformationConfiguration;
- try {
- facebookUserInformationConfiguration = mapper.readValue(config.root().render(ConfigRenderOptions.concise()), FacebookUserInformationConfiguration.class);
- } catch (IOException ex) {
- ex.printStackTrace();
- return;
- }
- this.klass = klass;
- }
-
- public FacebookUserstreamProvider(FacebookUserstreamConfiguration config, Class klass) {
- this.configuration = config;
- this.klass = klass;
- }
-
- public Queue<StreamsDatum> getProviderQueue() {
- return this.providerQueue;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
-
- client = getFacebookClient();
-
- if ( configuration.getInfo() != null
- &&
- configuration.getInfo().size() > 0 ) {
- for ( String id : configuration.getInfo()) {
- executor.submit(new FacebookFeedPollingTask(this, id));
- }
- running.set(true);
- } else {
- try {
- String id = client.getMe().getId();
- executor.submit(new FacebookFeedPollingTask(this, id));
- running.set(true);
- } catch (FacebookException ex) {
- LOGGER.error(ex.getMessage());
- running.set(false);
- }
- }
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- synchronized (FacebookUserstreamProvider.class) {
- current = new StreamsResultSet(new ConcurrentLinkedQueue<>(providerQueue));
- current.setCounter(new DatumStatusCounter());
- current.getCounter().add(countersCurrent);
- countersTotal.add(countersCurrent);
- countersCurrent = new DatumStatusCounter();
- providerQueue.clear();
- }
-
- return current;
-
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- LOGGER.debug("{} readNew", STREAMS_ID);
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- LOGGER.debug("{} readRange", STREAMS_ID);
- this.start = start;
- this.end = end;
- readCurrent();
- return (StreamsResultSet) providerQueue.iterator();
- }
-
- @Override
- public boolean isRunning() {
- return running.get();
- }
-
- void shutdownAndAwaitTermination(ExecutorService pool) {
- pool.shutdown(); // Disable new tasks from being submitted
- try {
- // Wait a while for existing tasks to terminate
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- pool.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- System.err.println("Pool did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- pool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- executor = newFixedThreadPoolWithQueueSize(5, 20);
-
- Objects.requireNonNull(providerQueue);
- Objects.requireNonNull(this.klass);
- Objects.requireNonNull(configuration.getOauth().getAppId());
- Objects.requireNonNull(configuration.getOauth().getAppSecret());
- Objects.requireNonNull(configuration.getOauth().getUserAccessToken());
-
- client = getFacebookClient();
-
- if ( configuration.getInfo() != null
- &&
- configuration.getInfo().size() > 0 ) {
-
- List<String> ids = new ArrayList<>();
- List<String[]> idsBatches = new ArrayList<>();
-
- for (String s : configuration.getInfo()) {
- if (s != null) {
- ids.add(s);
-
- if (ids.size() >= 100) {
- // add the batch
- idsBatches.add(ids.toArray(new String[ids.size()]));
- // reset the Ids
- ids = new ArrayList<>();
- }
-
- }
- }
- }
- }
-
- protected Facebook getFacebookClient() {
- ConfigurationBuilder cb = new ConfigurationBuilder();
- cb.setDebugEnabled(true)
- .setOAuthAppId(configuration.getOauth().getAppId())
- .setOAuthAppSecret(configuration.getOauth().getAppSecret())
- .setOAuthAccessToken(configuration.getOauth().getUserAccessToken())
- .setOAuthPermissions(ALL_PERMISSIONS)
- .setJSONStoreEnabled(true);
-
- FacebookFactory ff = new FacebookFactory(cb.build());
-
- return ff.getInstance();
- }
-
- @Override
- public void cleanUp() {
- shutdownAndAwaitTermination(executor);
- }
-
- private class FacebookFeedPollingTask implements Runnable {
-
- FacebookUserstreamProvider provider;
- Facebook client;
- String id;
-
- private Set<Post> priorPollResult = new HashSet<>();
-
- public FacebookFeedPollingTask(FacebookUserstreamProvider facebookUserstreamProvider) {
- this.provider = facebookUserstreamProvider;
- }
-
- public FacebookFeedPollingTask(FacebookUserstreamProvider facebookUserstreamProvider, String id) {
- this.provider = facebookUserstreamProvider;
- this.client = provider.client;
- this.id = id;
- }
-
- @Override
- public void run() {
- while (provider.isRunning()) {
- ResponseList<Post> postResponseList;
- try {
- postResponseList = client.getFeed(id);
-
- Set<Post> update = new HashSet<>(postResponseList);
- Set<Post> repeats = priorPollResult.stream().filter(update::contains).collect(Collectors.toSet());
- Set<Post> entrySet = update.stream().filter((x) -> !repeats.contains(x)).collect(Collectors.toSet());
- LOGGER.debug(this.id + " response: " + update.size() + " previous: " + repeats.size() + " new: " + entrySet.size());
- for (Post item : entrySet) {
- String json = DataObjectFactory.getRawJSON(item);
- org.apache.streams.facebook.Post post = mapper.readValue(json, org.apache.streams.facebook.Post.class);
- try {
- lock.readLock().lock();
- ComponentUtils.offerUntilSuccess(new StreamsDatum(post), providerQueue);
- countersCurrent.incrementAttempt();
- } finally {
- lock.readLock().unlock();
- }
- }
- priorPollResult = update;
- } catch (Exception ex) {
- ex.printStackTrace();
- } finally {
- try {
- Thread.sleep(configuration.getPollIntervalMillis());
- } catch (InterruptedException interrupt) {
- Thread.currentThread().interrupt();
- }
- }
- }
- }
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/page/FacebookPageDataCollector.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/page/FacebookPageDataCollector.java
deleted file mode 100644
index 5f697f7..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/page/FacebookPageDataCollector.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.provider.page;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.facebook.FacebookConfiguration;
-import org.apache.streams.facebook.IdConfig;
-import org.apache.streams.facebook.provider.FacebookDataCollector;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.annotations.VisibleForTesting;
-import facebook4j.FacebookException;
-import facebook4j.Page;
-import facebook4j.json.DataObjectFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.BlockingQueue;
-
-/**
- * Collects the page data from public Facebook pages.
- */
-public class FacebookPageDataCollector extends FacebookDataCollector {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookPageDataCollector.class);
- private static final int MAX_ATTEMPTS = 5;
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- public FacebookPageDataCollector(BlockingQueue<StreamsDatum> queue, FacebookConfiguration configuration) {
- super(configuration, queue);
- }
-
- @Override
- protected void getData(IdConfig id) throws Exception {
- Page responsePage = getPage(id.getId());
- backOff.reset();
-
- if (responsePage != null) {
- super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(responsePage), org.apache.streams.facebook.Page.class), responsePage.getId());
- }
- }
-
- protected Page getPage(String pageId) throws Exception {
- int attempt = 0;
- while (attempt < MAX_ATTEMPTS) {
- ++attempt;
- try {
- return getNextFacebookClient().getPage(pageId);
- } catch (FacebookException fe) {
- LOGGER.error("Facebook returned an exception : {}", fe);
- LOGGER.error("Facebook returned an exception while trying to get feed for page, {} : {}", pageId, fe.getMessage());
-
- int errorCode = fe.getErrorCode();
-
- //Some sort of rate limiting
- if (errorCode == 17 || errorCode == 4 || errorCode == 341) {
- super.backOff.backOff();
- }
- }
- }
- throw new Exception("Failed to get data from facebook after " + MAX_ATTEMPTS);
- }
-
- @VisibleForTesting
- protected BlockingQueue<StreamsDatum> getQueue() {
- return super.getQueue();
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/page/FacebookPageProvider.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/page/FacebookPageProvider.java
deleted file mode 100644
index ff57839..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/page/FacebookPageProvider.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.provider.page;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.facebook.FacebookConfiguration;
-import org.apache.streams.facebook.FacebookPageProviderConfiguration;
-import org.apache.streams.facebook.provider.FacebookDataCollector;
-import org.apache.streams.facebook.provider.FacebookProvider;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Streams Provider which collects Page Profiles in the ID List contained in the
- * FacebookConfiguration object
- *
- * <p/>
- * To use from command line,
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.facebook.provider.page.FacebookPageProvider -Dexec.args="application.conf pages.json"
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p>
- * facebook.oauth.appId
- * facebook.oauth.appSecret
- * facebook.oauth.userAccessToken
- * </p>
- */
-public class FacebookPageProvider extends FacebookProvider {
-
- public static final String STREAMS_ID = "FacebookPageProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookPageProvider.class);
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private FacebookPageProviderConfiguration configuration;
-
- public FacebookPageProvider(FacebookConfiguration facebookConfiguration) {
- super(facebookConfiguration);
- }
-
- @VisibleForTesting
- BlockingQueue<StreamsDatum> getQueue() {
- return super.datums;
- }
-
- @Override
- protected FacebookDataCollector getDataCollector() {
- return new FacebookPageDataCollector(super.datums, super.configuration);
- }
-
- /**
- * Run FacebookPageProvider from command line.
- * @param args configfile outfile
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File confFile = new File(configfile);
- assert (confFile.exists());
- Config conf = ConfigFactory.parseFileAnySyntax(confFile, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = conf.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- FacebookConfiguration config = new ComponentConfigurator<>(FacebookConfiguration.class).detectConfiguration(typesafe, "facebook");
- FacebookPageProvider provider = new FacebookPageProvider(config);
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- try {
- json = MAPPER.writeValueAsString(datum.getDocument());
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/pagefeed/FacebookPageFeedDataCollector.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/pagefeed/FacebookPageFeedDataCollector.java
deleted file mode 100644
index 5cf8b3e..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/pagefeed/FacebookPageFeedDataCollector.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.provider.pagefeed;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.facebook.FacebookConfiguration;
-import org.apache.streams.facebook.IdConfig;
-import org.apache.streams.facebook.provider.FacebookDataCollector;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import facebook4j.FacebookException;
-import facebook4j.Paging;
-import facebook4j.Post;
-import facebook4j.Reading;
-import facebook4j.ResponseList;
-import facebook4j.json.DataObjectFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.BlockingQueue;
-
-/**
- * Collects the page feed data from public Facebook pages.
- */
-public class FacebookPageFeedDataCollector extends FacebookDataCollector {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookPageFeedDataCollector.class);
- private static final int MAX_ATTEMPTS = 5;
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
- private static final int LIMIT = 100;
-
- public FacebookPageFeedDataCollector(BlockingQueue<StreamsDatum> queue, FacebookConfiguration configuration) {
- super(configuration, queue);
- }
-
- @Override
- protected void getData(IdConfig id) throws Exception {
- boolean exit = false;
-
- ResponseList<Post> facebookPosts = getPosts(id.getId());
- LOGGER.debug("Post received : {}", facebookPosts.size());
- backOff.reset();
- do {
- for (Post post : facebookPosts) {
- if (id.getBeforeDate() != null && id.getAfterDate() != null) {
- if (id.getBeforeDate().isAfter(post.getCreatedTime().getTime())
- && id.getAfterDate().isBefore(post.getCreatedTime().getTime())) {
- super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(post), org.apache.streams.facebook.Post.class), post.getId());
-
- }
- } else if (id.getBeforeDate() != null && id.getBeforeDate().isAfter(post.getCreatedTime().getTime())) {
- super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(post), org.apache.streams.facebook.Post.class), post.getId());
- } else if (id.getAfterDate() != null && id.getAfterDate().isBefore(post.getCreatedTime().getTime())) {
- super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(post), org.apache.streams.facebook.Post.class), post.getId());
- } else if (id.getBeforeDate() == null && id.getAfterDate() == null) {
- super.outputData(MAPPER.readValue(DataObjectFactory.getRawJSON(post), org.apache.streams.facebook.Post.class), post.getId());
- } else {
- exit = true;
- LOGGER.debug("Breaking on post, {}, with createdAtDate {}", post.getId(), post.getCreatedTime());
- break;
- }
- }
- if (facebookPosts.getPaging() != null && !exit) {
- LOGGER.debug("Paging. . .");
- facebookPosts = getPosts(facebookPosts.getPaging());
- backOff.reset();
- LOGGER.debug("Paging received {} posts*", facebookPosts.size());
- } else {
- LOGGER.debug("No more paging.");
- facebookPosts = null;
- }
- }
- while (facebookPosts != null && facebookPosts.size() != 0);
-
- }
-
- private ResponseList<Post> getPosts(Paging<Post> paging) throws Exception {
- return getPosts(null, paging);
- }
-
- private ResponseList<Post> getPosts(String pageId) throws Exception {
- return getPosts(pageId, null);
- }
-
- /**
- * Queries facebook. Attempts requests up to 5 times and backs off on each facebook exception.
- * @param pageId pageId
- * @param paging paging
- * @return ResponseList of $link{facebook4j.Post}
- * @throws Exception Exception
- */
- private ResponseList<Post> getPosts(String pageId, Paging<Post> paging) throws Exception {
- int attempt = 0;
- while (attempt < MAX_ATTEMPTS) {
- ++attempt;
- try {
- if (pageId != null) {
- Reading reading = new Reading();
- reading.limit(LIMIT);
- return getNextFacebookClient().getPosts(pageId, reading);
- } else {
- return getNextFacebookClient().fetchNext(paging);
- }
- } catch (FacebookException fe) {
- LOGGER.error("Facebook returned an exception : {}", fe);
- LOGGER.error("Facebook returned an exception while trying to get feed for page, {} : {}", pageId, fe.getMessage());
- //TODO Rate limit exceptions with facebook4j unclear http://facebook4j.org/oldjavadocs/1.1.12-2.0.0/2.0.0/index.html?facebook4j/internal/http/HttpResponseCode.html
- // back off at all exceptions until figured out.
- int errorCode = fe.getErrorCode();
-
- //Some sort of rate limiting
- if (errorCode == 17 || errorCode == 4 || errorCode == 341) {
- super.backOff.backOff();
- }
- }
- }
- throw new Exception("Failed to get data from facebook after " + MAX_ATTEMPTS);
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/pagefeed/FacebookPageFeedProvider.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/pagefeed/FacebookPageFeedProvider.java
deleted file mode 100644
index 3e7d8d9..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/provider/pagefeed/FacebookPageFeedProvider.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.provider.pagefeed;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.facebook.FacebookConfiguration;
-import org.apache.streams.facebook.provider.FacebookDataCollector;
-import org.apache.streams.facebook.provider.FacebookProvider;
-import org.apache.streams.facebook.provider.page.FacebookPageProvider;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.concurrent.TimeUnit;
-
-/**
- * FacebookPageFeedProvider provides content from facebook public page.
- */
-public class FacebookPageFeedProvider extends FacebookProvider {
-
- public static final String STREAMS_ID = "FacebookPageFeedProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookPageProvider.class);
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- public FacebookPageFeedProvider() {
- super();
- }
-
- public FacebookPageFeedProvider(FacebookConfiguration config) {
- super(config);
- }
-
- @Override
- protected FacebookDataCollector getDataCollector() {
- return new FacebookPageFeedDataCollector(super.datums, super.configuration);
- }
-
- /**
- * Run FacebookPageFeedProvider from command line.
- * @param args configfile outfile
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File confFile = new File(configfile);
- assert (confFile.exists());
- Config conf = ConfigFactory.parseFileAnySyntax(confFile, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = conf.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- FacebookConfiguration config = new ComponentConfigurator<>(FacebookConfiguration.class).detectConfiguration(typesafe, "facebook");
- FacebookPageFeedProvider provider = new FacebookPageFeedProvider(config);
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- try {
- json = MAPPER.writeValueAsString(datum.getDocument());
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/serializer/FacebookActivityUtil.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/serializer/FacebookActivityUtil.java
deleted file mode 100644
index dc7aa52..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/serializer/FacebookActivityUtil.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.serializer;
-
-import org.apache.streams.data.util.ActivityUtil;
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.facebook.Cover;
-import org.apache.streams.facebook.Datum;
-import org.apache.streams.facebook.Like;
-import org.apache.streams.facebook.Location;
-import org.apache.streams.facebook.Page;
-import org.apache.streams.facebook.Place;
-import org.apache.streams.facebook.Post;
-import org.apache.streams.facebook.To;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.pojo.json.Provider;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * FacebookActivityUtil helps convert facebook data to activity formats.
- */
-public class FacebookActivityUtil {
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookActivityUtil.class);
-
- /**
- * Updates the given Activity object with the values from the Page.
- * @param page the object to use as the source
- * @param activity the target of the updates. Will receive all values from the Page.
- * @throws ActivitySerializerException
- */
- public static void updateActivity(Page page, Activity activity) throws ActivitySerializerException {
- activity.setActor(buildActor(page));
- activity.setId(null);
- activity.setProvider(getProvider());
- }
-
- /**
- * Updates the given Activity object with the values from the Post.
- * @param post post
- * @param activity activity
- * @throws ActivitySerializerException
- */
- public static void updateActivity(Post post, Activity activity) throws ActivitySerializerException {
- activity.setActor(buildActor(post));
- activity.setId(formatId(post.getId()));
- activity.setProvider(getProvider());
- activity.setUpdated(post.getUpdatedTime());
- activity.setPublished(post.getCreatedTime());
-
- if (post.getLink() != null && post.getLink().length() > 0) {
- List<String> links = new ArrayList<>();
- links.add(post.getLink());
- activity.setLinks(links);
- }
-
- activity.setContent(post.getMessage());
-
- activity.setVerb("post");
- activity.setObject(buildObject(post));
- buildExtensions(activity, post);
- }
-
- /**
- * Builds out the {@link ActivityObject} from the given {@link Post}.
- * @param post
- * @return {@link ActivityObject}
- */
- public static ActivityObject buildObject(Post post) {
- ActivityObject activityObject = new ActivityObject();
-
- try {
- activityObject.setContent(post.getMessage());
- activityObject.setPublished(post.getCreatedTime());
- activityObject.setUpdated(post.getUpdatedTime());
- activityObject.setDisplayName(post.getFrom().getName());
- activityObject.setId(formatId(post.getId()));
- activityObject.setObjectType(post.getType());
- activityObject.setUrl(post.getLink());
-
- if (activityObject.getObjectType().equals("photo")) {
- Image image = new Image();
- image.setUrl(activityObject.getUrl());
- activityObject.setImage(image);
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to build Activity object for post: {}, exception: {}", post, ex);
- }
-
- return activityObject;
- }
-
- /**
- * Gets the common facebook {@link Provider} object.
- * @return a provider object representing Facebook
- */
- public static Provider getProvider() {
- Provider provider = new Provider();
- provider.setId("id:providers:facebook");
- provider.setDisplayName("Facebook");
-
- return provider;
- }
-
- /**
- * Builds the activity {@link ActivityObject} actor from the Page.
- * @param page the object to use as the source
- * @return a valid Actor populated from the Page
- */
- public static ActivityObject buildActor(Page page) {
- ActivityObject actor = new ActivityObject();
- actor.setId(formatId(
- Optional.ofNullable(Optional.ofNullable(page.getId())
- .orElseGet(Optional.of(page.getId())::get)).orElse(null)
- ));
-
- actor.setDisplayName(page.getName());
- actor.setAdditionalProperty("handle", page.getUsername());
- actor.setSummary(page.getAbout());
-
- if (page.getLink() != null) {
- actor.setUrl(page.getLink());
- }
-
- Image profileImage = new Image();
- Cover cover = page.getCover();
-
- if (cover != null) {
- profileImage.setUrl(cover.getSource());
- }
- actor.setImage(profileImage);
-
- buildExtensions(actor, page);
-
- return actor;
- }
-
- /**
- * Builds an {@link ActivityObject} object from the {@link Post}.
- * @param post post
- * @return {@link ActivityObject}
- */
- public static ActivityObject buildActor(Post post) {
- ActivityObject actor = new ActivityObject();
-
- try {
- actor.setId(formatId(
- Optional.ofNullable(Optional.ofNullable(post.getFrom().getId())
- .orElseGet(Optional.of(post.getFrom().getId())::get)).orElse(null)
- ));
-
- actor.setDisplayName(post.getFrom().getName());
- actor.setAdditionalProperty("handle", post.getFrom().getName());
- } catch (Exception ex) {
- LOGGER.error("Exception trying to build actor for Post: {}, {}", post, ex);
- }
-
- return actor;
- }
-
- /**
- * Builds the actor extensions given the page object.
- * @param actor actor
- * @param page page
- */
- public static void buildExtensions(ActivityObject actor, Page page) {
- Map<String, Object> extensions = new HashMap<>();
- Location location = page.getLocation();
-
- if (location != null) {
- extensions.put("location", page.getLocation().toString());
- }
-
- extensions.put("favorites", page.getTalkingAboutCount());
- extensions.put("followers", page.getFanCount());
-
- extensions.put("facebook", page);
-
- actor.setAdditionalProperty("extensions", extensions);
- }
-
- /**
- * Fills out the extensions attribute of the passed in {@link Activity}
- * @param activity
- * @param post
- */
- @SuppressWarnings("unchecked")
- public static void buildExtensions(Activity activity, Post post) {
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
-
- if(post.getLikes() != null && post.getLikes().size() > 0) {
- Map<String, Object> likes = new HashMap<>();
- Like like = post.getLikes().get(0);
-
- if(like.getAdditionalProperties().containsKey("data")) {
- extensions.put("likes", likes);
- }
- }
-
- if(post.getShares() != null) {
- Map<String, Object> shares = new HashMap<>();
- shares.put("count", ((Map<String, Object>)post.getShares()).get("count"));
- extensions.put("rebroadcasts", shares);
- }
-
- if(post.getTo() != null) {
- To to = post.getTo();
- List<Datum> data = to.getData();
- extensions.put("user_mentions", new ArrayList<>());
-
- for(Datum d : data) {
- Map<String, String> mention = new HashMap<>();
-
- mention.put("id", d.getId());
- mention.put("displayName", d.getName());
- mention.put("handle", d.getName());
-
- ((List<Map<String,String>>)extensions.get("user_mentions")).add(mention);
- }
- }
-
- if(post.getPlace() != null) {
- Place place = post.getPlace();
- if(place.getAdditionalProperties().containsKey("location")) {
- extensions.put(ActivityUtil.LOCATION_EXTENSION, place.getAdditionalProperties().get("location"));
- }
- }
-
- extensions.put("facebook", mapper.convertValue(post, ObjectNode.class));
- }
-
- /**
- * Formats the ID to conform with the Apache Streams activity ID convention
- * @param idparts the parts of the ID to join
- * @return a valid Activity ID in format "id:facebook:part1:part2:...partN"
- */
- public static String formatId(String... idparts) {
- return String.join(":",
- Stream.concat(Arrays.stream(new String[]{"id:facebook"}), Arrays.stream(idparts)).collect(Collectors.toList()));
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/serializer/FacebookStreamsPostSerializer.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/serializer/FacebookStreamsPostSerializer.java
deleted file mode 100644
index 772af83..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/serializer/FacebookStreamsPostSerializer.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.serializer;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.facebook.Post;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Provider;
-
-import java.util.List;
-
-
-/**
- * Converts {@link org.apache.streams.facebook.Post} to {@link org.apache.streams.pojo.json.Activity}
- */
-public class FacebookStreamsPostSerializer implements ActivitySerializer<Post> {
-
- private static final String FACEBOOK_STREAMS_ID = "id:provider:facebook";
- private static final String ID_PREFIX = "id:facebook:";
- private static final String PROVIDER_DISPLAY = "Facebook";
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public Post serialize(Activity deserialized) throws ActivitySerializerException {
- return null;
- }
-
- @Override
- public Activity deserialize(Post post) throws ActivitySerializerException {
- Activity activity = new Activity();
- activity.setActor(createActor(post));
-
- activity.setId(post.getId());
- activity.setContent(post.getMessage());
- return null;
- }
-
- @Override
- public List<Activity> deserializeAll(List<Post> serializedList) {
- return null;
- }
-
- /**
- * createActor.
- * @param post post
- * @return ActivityObject
- */
- public ActivityObject createActor(Post post) {
- ActivityObject actor = new ActivityObject();
- actor.setDisplayName(post.getFrom().getName());
- actor.setId(ID_PREFIX + post.getFrom().getId());
- return actor;
- }
-
- /**
- * Provider.
- * @param post post
- * @return Provider
- */
- public Provider createProvider(Post post) {
- Provider provider = new Provider();
- provider.setId(FACEBOOK_STREAMS_ID);
- provider.setDisplayName(PROVIDER_DISPLAY);
- return provider;
- }
-
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/FacebookConfiguration.json b/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/FacebookConfiguration.json
deleted file mode 100644
index cab7c1b..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/FacebookConfiguration.json
+++ /dev/null
@@ -1,85 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.facebook.FacebookConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "protocol": {
- "type": "string",
- "description": "The protocol"
- },
- "host": {
- "type": "string",
- "description": "The host"
- },
- "port": {
- "type": "integer",
- "description": "The port"
- },
- "version": {
- "type": "string",
- "description": "The version"
- },
- "endpoint": {
- "type": "string",
- "description": "The endpoint"
- },
- "oauth": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.facebook.FacebookOAuthConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "appId": {
- "type": "string"
- },
- "appSecret": {
- "type": "string"
- },
- "appAccessToken": {
- "type": "string"
- },
- "userAccessToken": {
- "type": "string"
- }
- }
- },
- "userAccessTokens": {
- "type": "array",
- "uniqueItems": true,
- "items": {
- "type": "string",
- "description": "User access tokens"
- },
- "description": "User access tokens that have been authorized for this app"
- },
- "ids": {
- "type": "array",
- "uniqueItems": true,
- "items": {
- "type": "object",
- "description": "id for the application to use. user id, page id, etc. with date ranges",
- "javaType": "org.apache.streams.facebook.IdConfig",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "type": "string"
- },
- "beforeDate": {
- "type": "string",
- "format": "date-time"
- },
- "afterDate": {
- "type": "string",
- "format": "date-time"
- }
- }
- },
- "description": "ids for the application to use. user ids, page ids, etc."
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/FacebookPageProviderConfiguration.json b/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/FacebookPageProviderConfiguration.json
deleted file mode 100644
index 8450cff..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/FacebookPageProviderConfiguration.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.facebook.FacebookPageProviderConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {"$ref":"FacebookConfiguration.json"},
- "properties": {
- "fields": {
- "type": "array",
- "description": "A list of user IDs, indicating users of interest",
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/FacebookUserInformationConfiguration.json b/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/FacebookUserInformationConfiguration.json
deleted file mode 100644
index e9dea97..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/FacebookUserInformationConfiguration.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.facebook.FacebookUserInformationConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {"$ref":"FacebookConfiguration.json"},
- "properties": {
- "info": {
- "type": "array",
- "description": "A list of user IDs, indicating users of interest",
- "items": {
- "type": "string"
- }
- },
- "pollIntervalMillis": {
- "type": "integer",
- "default" : "60000",
- "description": "Polling interval in ms"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/FacebookUserstreamConfiguration.json b/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/FacebookUserstreamConfiguration.json
deleted file mode 100644
index 86fc484..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/FacebookUserstreamConfiguration.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.facebook.FacebookUserstreamConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {"$ref":"FacebookConfiguration.json"},
- "properties": {
- "info": {
- "type": "array",
- "description": "A list of user IDs, indicating users of interest",
- "items": {
- "type": "string"
- }
- },
- "pollIntervalMillis": {
- "type": "integer",
- "default" : "60000",
- "description": "Polling interval in ms"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/Page.json b/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/Page.json
deleted file mode 100644
index 72f96f3..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/Page.json
+++ /dev/null
@@ -1,281 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.facebook.Page",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "about": {
- "type": "string",
- "required": false
- },
- "username": {
- "type": "string",
- "required": false
- },
- "accessToken": {
- "type": "string",
- "required": false
- },
- "birthday": {
- "type": "string",
- "required": false
- },
- "category": {
- "type": "string",
- "required": false
- },
- "can_checkin": {
- "type": "boolean",
- "required": false
- },
- "can_post": {
- "type": "boolean",
- "required": false
- },
- "checkins": {
- "type": "integer",
- "required": false
- },
- "communityPage": {
- "type": "boolean",
- "required": false
- },
- "cover": {
- "type": "object",
- "required": false,
- "properties": {
- "id": {
- "type": "null",
- "required": false
- },
- "offsetY": {
- "type": "number",
- "required": false
- },
- "source": {
- "type": "string",
- "required": false
- }
- }
- },
- "display_subtext": {
- "type": "string",
- "required": false
- },
- "displayed_message_response_time": {
- "type": "string",
- "required": false
- },
- "engagement": {
- "type": "object",
- "required": false,
- "properties": {
- "count": {
- "type": "integer",
- "required": false
- },
- "social_sentence": {
- "type": "string",
- "required": false
- }
- }
- },
- "fan_count": {
- "type": "integer",
- "required": false
- },
- "founded": {
- "type": "string",
- "required": false
- },
- "general_info": {
- "type": "string",
- "required": false
- },
- "global_brand_root_id": {
- "type": "string",
- "required": false
- },
- "id": {
- "type": "string",
- "required": false
- },
- "is_community_page": {
- "type": "boolean",
- "required": false
- },
- "is_always_open": {
- "type": "boolean",
- "required": false
- },
- "is_permanently_closed": {
- "type": "boolean",
- "required": false
- },
- "is_published": {
- "type": "boolean",
- "required": false
- },
- "is_unclaimed": {
- "type": "boolean",
- "required": false
- },
- "is_webhooks_subscribed": {
- "type": "boolean",
- "required": false
- },
- "leadgen_tos_accepted": {
- "type": "boolean",
- "required": false
- },
- "link": {
- "type": "string",
- "required": false
- },
- "location": {
- "type": "object",
- "required": false,
- "properties": {
- "city": {
- "type": "string",
- "required": false
- },
- "country": {
- "type": "string",
- "required": false
- },
- "latitude": {
- "type": "null",
- "required": false
- },
- "longitude": {
- "type": "null",
- "required": false
- },
- "state": {
- "type": "string",
- "required": false
- },
- "street": {
- "type": "string",
- "required": false
- },
- "text": {
- "type": "null",
- "required": false
- },
- "zip": {
- "type": "string",
- "required": false
- }
- }
- },
- "mission": {
- "type": "string",
- "required": false
- },
- "parking": {
- "type": "object",
- "required": false,
- "properties": {
- "lot": {
- "type": "integer",
- "required": false
- },
- "street": {
- "type": "integer",
- "required": false
- },
- "valet": {
- "type": "integer",
- "required": false
- }
- }
- },
- "name": {
- "type": "string",
- "id": "http://jsonschema.net/name",
- "required": false
- },
- "name_with_location_descriptor": {
- "type": "string",
- "required": false
- },
- "overall_star_rating": {
- "type": "integer",
- "required": false
- },
- "phone": {
- "type": "string",
- "required": false
- },
- "picture": {
- "type": "null",
- "required": false
- },
- "published": {
- "type": "boolean",
- "required": false
- },
- "rating_count": {
- "type": "integer",
- "required": false
- },
- "talking_about_count": {
- "type": "integer",
- "required": false
- },
- "voip_info": {
- "type": "object",
- "required": false,
- "properties": {
- "has_mobile_app": {
- "type": "boolean",
- "required": false
- },
- "has_permission": {
- "type": "boolean",
- "required": false
- },
- "is_callable": {
- "type": "boolean",
- "required": false
- },
- "is_callable_webrtc": {
- "type": "boolean",
- "required": false
- },
- "is_pushable": {
- "type": "boolean",
- "required": false
- },
- "reason_code": {
- "type": "integer",
- "required": false
- },
- "reason_description": {
- "type": "string",
- "required": false
- }
- }
- },
- "verification_status": {
- "type": "string",
- "required": false
- },
- "website": {
- "type": "string",
- "required": false
- },
- "were_here_count": {
- "type": "integer",
- "required": false
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/graph/Post.json b/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/graph/Post.json
deleted file mode 100644
index 487013c..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/jsonschema/org/apache/streams/facebook/graph/Post.json
+++ /dev/null
@@ -1,207 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.facebook.Post",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "type": "string"
- },
- "from": {
- "type": "object",
- "properties": {
- "id": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "to": {
- "type": "object",
- "properties": {
- "data": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "id": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- }
- }
- }
- },
- "message": {
- "type": "string"
- },
- "message_tags": {
- "type": "object",
- "properties": {
- "data": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "id": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- }
- }
- }
- },
- "picture": {
- "type": "string"
- },
- "link": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "caption": {
- "type": "string"
- },
- "description": {
- "type": "string"
- },
- "source": {
- "type": "string"
- },
- "icon": {
- "type": "string"
- },
- "actions": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "name": {
- "type": "string"
- },
- "link": {
- "type": "string"
- }
- }
- }
- },
- "comments": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "id": {
- "type": "string"
- },
- "from": {
- "type": "string"
- },
- "message": {
- "type": "string"
- },
- "created_time": {
- "type": "string",
- "format" : "date-time"
- }
- }
- }
- },
- "likes": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "name": {
- "type": "string"
- },
- "link": {
- "type": "string"
- }
- }
- }
- },
- "type": {
- "type": "string"
- },
- "place": {
- "type": "object",
- "properties": {
- "name": {
- "type": "string"
- },
- "id": {
- "type": "string"
- }
- }
- },
- "story": {
- "type": "string"
- },
- "shares": {
- "type": "int"
- },
- "object_id": {
- "type": "int"
- },
- "application": {
- "type": "object",
- "properties": {
- "name": {
- "type": "string"
- },
- "id": {
- "type": "string"
- }
- }
- },
- "created_time": {
- "type": "string",
- "format" : "date-time"
- },
- "updated_time": {
- "type": "string",
- "format" : "date-time"
- },
- "include_hidden": {
- "type": "boolean"
- },
- "status_type": {
- "type": "string"
- },
- "properties": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "name": {
- "type": "string"
- },
- "text": {
- "type": "string"
- }
- }
- }
- },
- "privacy": {
- "type": "object",
- "properties": {
- "value": {
- "type": "string"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/main/resources/components.dot b/streams-contrib/streams-provider-facebook/src/main/resources/components.dot
deleted file mode 100644
index 9dccaa3..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/resources/components.dot
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- subgraph cluster_generators {
- label="generators";
- style="dotted";
- generators_facebook [label="https://api.facebook.com"]
- }
-
- subgraph cluster_providers {
- label="providers";
- providers_facebook_page [label="FacebookPageProvider"]
- providers_facebook_pagefeed [label="FacebookPageFeedDataCollector"]
- }
-
- subgraph cluster_processors {
- label="processors";
- processors_facebook_activity [label="FacebookPageActivitySerializer"]
- processors_facebook_activityobject [label="FacebookPostActivitySerializer"]
- }
-
- persisters [label="persisters", shape="cylinder"];
-
- generators_facebook -> providers_facebook_page
- generators_facebook -> providers_facebook_pagefeed
- providers_facebook_page -> processors_facebook_activityobject [label="o.a.s.f.Page"]
- providers_facebook_pagefeed -> processors_facebook_activity [label="o.a.s.f.Post\no.a.s.f.Comment"]
-
- processors_facebook_activityobject -> persisters [label="o.a.s.p.j.Page"]
- processors_facebook_activity -> persisters [label="o.a.s.p.j.Post\no.a.s.p.j.Comment"]
-
-}
diff --git a/streams-contrib/streams-provider-facebook/src/main/resources/reference.conf b/streams-contrib/streams-provider-facebook/src/main/resources/reference.conf
deleted file mode 100644
index eda1404..0000000
--- a/streams-contrib/streams-provider-facebook/src/main/resources/reference.conf
+++ /dev/null
@@ -1,134 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-facebook.version = "v2.8"
-facebook.fields = [
- "id"
- "about"
- "access_token"
-// "ad_campaign"
- "affiliation"
- "app_id"
- "app_links"
- "artists_we_like"
- "attire"
- "awards"
- "band_interests"
- "band_members"
- "best_page"
- "bio"
- "birthday"
- "booking_agent"
- "built"
- "can_checkin"
- "can_post"
- "category"
- "category_list"
- "checkins"
- "company_overview"
- "contact_address"
- "context"
- "country_page_likes"
- "cover"
- "culinary_team"
- "current_location"
- "description"
- "description_html"
- "directed_by"
- "display_subtext"
- "displayed_message_response_time"
- "emails"
- "engagement"
- "fan_count"
- "featured_video"
- "features"
- "food_styles"
- "founded"
- "general_info"
- "general_manager"
- "genre"
- "global_brand_page_name"
- "global_brand_root_id"
- "has_added_app"
- "hometown"
- "hours"
- "impressum"
- "influences"
-// "instant_articles_review_status"
- "is_always_open"
- "is_community_page"
- "is_permanently_closed"
- "is_published"
- "is_unclaimed"
- "is_verified"
- "is_webhooks_subscribed"
- "leadgen_tos_accepted"
- "link"
- "location"
- "members"
- "mission"
- "mpg"
- "name"
- "name_with_location_descriptor"
- "network"
- "new_like_count"
- "offer_eligible"
- "overall_star_rating"
- "parent_page"
- "parking"
- "payment_options"
- "personal_info"
- "personal_interests"
- "pharma_safety_info"
- "phone"
- "place_type"
- "plot_outline"
-// "preferred_audience"
- "press_contact"
- "price_range"
- "produced_by"
- "products"
-// "promotion_eligible"
-// "promotion_ineligible_reason"
- "public_transit"
- "publisher_space"
- "rating_count"
-// "recipient"
- "record_label"
- "release_date"
- "restaurant_services"
- "restaurant_specialties"
- "schedule"
- "screenplay_by"
- "season"
- "single_line_address"
- "starring"
-// "start_info"
- "store_location_descriptor"
- "store_number"
- "studio"
-// "supports_instant_articles"
- "talking_about_count"
- "unread_message_count"
- "unread_notif_count"
- "unseen_message_count"
- "username"
- "verification_status"
- "voip_info"
- "website"
- "were_here_count"
- "written_by"
-]
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/site/markdown/index.md b/streams-contrib/streams-provider-facebook/src/site/markdown/index.md
deleted file mode 100644
index b80f2f0..0000000
--- a/streams-contrib/streams-provider-facebook/src/site/markdown/index.md
+++ /dev/null
@@ -1,51 +0,0 @@
-streams-provider-facebook
-==============
-
-streams-provider-facebook contains schema definitions, providers, conversions, and utility classes.
-
-## Data Types
-
-[Page.json](org/apache/streams/facebook/Page.json "Page.json")
-
-[Post.json](org/apache/streams/facebook/graph/Post.json "Post.json")
-
-## Configuration
-
-| Schema |
-|--------|
-| [FacebookConfiguration.json](org/apache/streams/facebook/FacebookConfiguration.json "FacebookConfiguration.json") [FacebookConfiguration.html](apidocs/org/apache/streams/facebook/FacebookConfiguration.html "javadoc") |
-
-## Components
-
-
-
-Test:
------
-
-Create a local file `facebook.conf` with valid twitter credentials
-
- facebook {
- oauth {
- appId = ""
- appSecret = ""
- }
- userAccessTokens = [
- ""
- ]
- }
-
-Build with integration testing enabled, using your credentials
-
- mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/facebook.conf"
-
-Confirm that you can get data from testing endpoints with Graph API Explorer:
-
- - ${page-id}
- - ${page-id}/feed
-
-[https://developers.facebook.com/tools/explorer/](https://developers.facebook.com/tools/explorer/ "https://developers.facebook.com/tools/explorer/")
-
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/FacebookEventClassifierTest.java b/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/FacebookEventClassifierTest.java
deleted file mode 100644
index 9d7604a..0000000
--- a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/FacebookEventClassifierTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.test;
-
-import org.apache.streams.facebook.Page;
-import org.apache.streams.facebook.Post;
-import org.apache.streams.facebook.provider.FacebookEventClassifier;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class FacebookEventClassifierTest {
-
- private String empty = "";
- private String def = "{}";
- private String post = "{\"metadata\":null,\"id\":\"687664828_10153082499914829\",\"from\":{\"metadata\":null,\"id\":\"687664828\",\"name\":\"Steve Blackmon\",\"category\":null,\"createdTime\":null},\"message\":null,\"picture\":\"https://fbexternal-a.akamaihd.net/app_full_proxy.php?app=184136951108&v=1&size=z&cksum=7f4094dff37cedd69072cd2c0b3728b7&src=https%3A%2F%2Fstatic.tripit.com%2Fimages%2Fplaces%2Fsantamonica.jpg%3Fv%3D2014-08-13\",\"link\":\"http://www.tripit.com/trip/show/id/1303 [...]
- private String page = "{\"metadata\":null,\"id\":\"142803045874943\",\"name\":\"Senator Angus S. King, Jr.\",\"category\":\"Government official\",\"createdTime\":null,\"link\":\"https://www.facebook.com/SenatorAngusSKingJr\",\"likes\":10246,\"location\":{\"street\":\"359 Dirksen Senate Office Building\",\"city\":\"Washington, District of Columbia\",\"state\":\"DC\",\"country\":\"United States\",\"zip\":\"20510\",\"latitude\":null,\"longitude\":null,\"text\":null},\"phone\":\"202-224-53 [...]
-
- @Test(expected=IllegalArgumentException.class)
- public void emptyJSONTest() {
- Class inClass = FacebookEventClassifier.detectClass(empty);
- }
-
- @Test
- public void defaultDetectTest() {
- Class inClass = FacebookEventClassifier.detectClass(post);
- assertEquals(inClass, Post.class);
- }
-
- @Test
- public void postDetectTest() {
- Class inClass = FacebookEventClassifier.detectClass(post);
- assertEquals(inClass, Post.class);
- }
-
- @Test
- public void pageDetectTest() {
- Class inClass = FacebookEventClassifier.detectClass(page);
- assertEquals(inClass, Page.class);
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/SimplePageTest.java b/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/SimplePageTest.java
deleted file mode 100644
index b7009e2..0000000
--- a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/SimplePageTest.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.test;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.facebook.Page;
-import org.apache.streams.facebook.api.FacebookPageActivitySerializer;
-import org.apache.streams.facebook.processor.FacebookTypeConverter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.junit.Assert.assertThat;
-
-/**
- * Tests conversion of Facebook Page inputs to Actor
- */
-@Ignore("ignore until test resources are available.")
-public class SimplePageTest {
- private static final Logger LOGGER = LoggerFactory.getLogger(SimplePageTest.class);
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- private ObjectNode event;
-
- private static final String FACEBOOK_JSON= "{\"about\":\"The Facebook Page celebrates how our friends inspire us, support us, and help us discover the world when we connect.\",\"username\":\"facebook\",\"birthday\":\"02/04/2004\",\"category\":\"Product/Service\",\"can_checkin\":true,\"can_post\":false,\"checkins\":12,\"cover\":{\"id\":\"10154345508521729\",\"source\":\"https://scontent.xx.fbcdn.net/v/t1.0-9/s720x720/12573693_10154345508521729_8347370496501004621_n.png?oh=b75f9ec17e7e [...]
-
- private FacebookPageActivitySerializer facebookPageActivitySerializer = new FacebookPageActivitySerializer();
-
- @Before
- public void setUp() throws Exception {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = SimplePageTest.class.getResourceAsStream("/testpage.json");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- event = null;
- event = (ObjectNode) mapper.readTree(FACEBOOK_JSON);
- }
-
- @Test
- public void TestSerialization() {
- assertThat(event, is(not(nullValue())));
-
- Page page = mapper.convertValue(event, Page.class);
-
- assertThat(page, is(not(nullValue())));
- assertThat(page.getAbout(), is(not(nullValue())));
- assertThat(page.getFanCount(), is(not(nullValue())));
- assertThat(page.getTalkingAboutCount(), is(not(nullValue())));
- }
-
- @Test
- public void TestDeserialization() throws Exception {
- Page page = mapper.convertValue(event, Page.class);
-
- Activity activity;
- activity = facebookPageActivitySerializer.deserialize(page);
-
- assertThat(activity, is(not(nullValue())));
-
- assertThat(activity.getActor(), is(not(nullValue())));
- assertThat(activity.getActor().getId(), is(not(nullValue())));
- assertThat(activity.getVerb(), is(not(nullValue())));
- assertThat(activity.getProvider(), is(not(nullValue())));
- }
-
- @Test
- public void TestConverter() throws Exception {
- FacebookTypeConverter converter = new FacebookTypeConverter(String.class, Activity.class);
- converter.prepare(null);
- converter.process(new StreamsDatum(FACEBOOK_JSON));
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/TestPage.java b/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/TestPage.java
deleted file mode 100644
index 3ebd133..0000000
--- a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/TestPage.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.streams.facebook.test;
-
-import facebook4j.Cover;
-import facebook4j.Page;
-import facebook4j.Place;
-
-import java.net.URL;
-import java.util.Date;
-import java.util.Map;
-
-public class TestPage implements Page {
- private String id;
- private String name;
-
- public TestPage(String id, String name) {
- this.id = id;
- this.name = name;
- }
-
- @Override
- public String getId() {
- return id;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public URL getLink() {
- return null;
- }
-
- @Override
- public String getCategory() {
- return null;
- }
-
- @Override
- public Boolean isPublished() {
- return null;
- }
-
- @Override
- public Boolean canPost() {
- return null;
- }
-
- @Override
- public Integer getLikes() {
- return null;
- }
-
- @Override
- public Place.Location getLocation() {
- return null;
- }
-
- @Override
- public String getPhone() {
- return null;
- }
-
- @Override
- public Integer getCheckins() {
- return null;
- }
-
- @Override
- public URL getPicture() {
- return null;
- }
-
- @Override
- public Cover getCover() {
- return null;
- }
-
- @Override
- public String getWebsite() {
- return null;
- }
-
- @Override
- public String getCompanyOverview() {
- return null;
- }
-
- @Override
- public Integer getTalkingAboutCount() {
- return null;
- }
-
- @Override
- public String getAccessToken() {
- return null;
- }
-
- @Override
- public Boolean isCommunityPage() {
- return null;
- }
-
- @Override
- public Integer getWereHereCount() {
- return null;
- }
-
- @Override
- public Integer getFanCount() {
- return null;
- }
-
- @Override
- public Date getCreatedTime() {
- return null;
- }
-
- @Override
- public String getAbout() {
- return null;
- }
-
- @Override
- public String getUsername() {
- return null;
- }
-
- @Override
- public String getMission() {
- return null;
- }
-
- @Override
- public Map<String, String> getHours() {
- return null;
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/data/FacebookActivityActorSerDeIT.java b/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/data/FacebookActivityActorSerDeIT.java
deleted file mode 100644
index 2ae4297..0000000
--- a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/data/FacebookActivityActorSerDeIT.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.test.data;
-
-import org.apache.streams.facebook.Page;
-import org.apache.streams.facebook.api.FacebookPageActivitySerializer;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.input.BoundedInputStream;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.InputStream;
-import java.nio.charset.Charset;
-
-/**
- * Tests conversion of Facebook Page inputs to Actor
- */
-public class FacebookActivityActorSerDeIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookActivityActorSerDeIT.class);
- private FacebookPageActivitySerializer serializer = new FacebookPageActivitySerializer();
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Test
- public void Tests() throws Exception
- {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = FacebookActivityActorSerDeIT.class.getResourceAsStream("/testpage.json");
- is = new BoundedInputStream(is, 10000);
- String json = String.join(" ", IOUtils.readLines(is, Charset.defaultCharset()));
- LOGGER.debug(json);
-
- Page page = mapper.readValue(json, Page.class);
-
- Activity activity = serializer.deserialize(page);
-
- LOGGER.debug(mapper.writeValueAsString(activity));
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/data/FacebookActivitySerDeIT.java b/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/data/FacebookActivitySerDeIT.java
deleted file mode 100644
index d1ca09a..0000000
--- a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/data/FacebookActivitySerDeIT.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.test.data;
-
-import org.apache.streams.facebook.Post;
-import org.apache.streams.facebook.api.FacebookPostActivitySerializer;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.input.BoundedInputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.InputStream;
-import java.nio.charset.Charset;
-
-/**
- * Tests conversion of Facebook Post inputs to Activity
- */
-public class FacebookActivitySerDeIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookActivitySerDeIT.class);
- private FacebookPostActivitySerializer serializer = new FacebookPostActivitySerializer();
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Test
- public void Tests()
- {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = FacebookActivitySerDeIT.class.getResourceAsStream("/testpost.json");
- is = new BoundedInputStream(is, 10000);
- String json;
-
- try {
- json = String.join(" ", IOUtils.readLines(is, Charset.defaultCharset()));
- LOGGER.debug(json);
-
- Post post = mapper.readValue(json, Post.class);
-
- Activity activity = serializer.deserialize(post);
-
- LOGGER.debug(mapper.writeValueAsString(activity));
-
- } catch( Exception e ) {
- LOGGER.error("Exception: ", e);
- Assert.fail();
- }
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/data/FacebookPageSerDeIT.java b/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/data/FacebookPageSerDeIT.java
deleted file mode 100644
index 7cdb54d..0000000
--- a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/data/FacebookPageSerDeIT.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.test.data;
-
-import org.apache.streams.facebook.Page;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.input.BoundedInputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.InputStream;
-import java.nio.charset.Charset;
-
-/**
- * Tests serialization of Facebook Page inputs
- */
-public class FacebookPageSerDeIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookPageSerDeIT.class);
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Test
- public void Tests()
- {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = FacebookPageSerDeIT.class.getResourceAsStream("/testpage.json");
- is = new BoundedInputStream(is, 10000);
- String json;
-
- try {
- json = String.join(" ", IOUtils.readLines(is, Charset.defaultCharset()));
- LOGGER.debug(json);
-
- Page ser = mapper.readValue(json, Page.class);
-
- String de = mapper.writeValueAsString(ser);
-
- LOGGER.debug(de);
-
- Page serde = mapper.readValue(de, Page.class);
-
- Assert.assertEquals(ser, serde);
-
- LOGGER.debug(mapper.writeValueAsString(serde));
-
- } catch( Exception e ) {
- LOGGER.error("Exception: ", e);
- Assert.fail();
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/data/FacebookPostSerDeIT.java b/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/data/FacebookPostSerDeIT.java
deleted file mode 100644
index a9877d8..0000000
--- a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/data/FacebookPostSerDeIT.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.test.data;
-
-import org.apache.streams.facebook.Post;
-import org.apache.streams.facebook.serializer.FacebookActivityUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.input.BoundedInputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.InputStream;
-import java.nio.charset.Charset;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-
-/**
- * Tests serialization of Facebook Post inputs
- */
-public class FacebookPostSerDeIT {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(FacebookPostSerDeIT.class);
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Test
- public void Tests()
- {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = FacebookPostSerDeIT.class.getResourceAsStream("/testpost.json");
- is = new BoundedInputStream(is, 10000);
- String json;
-
- try {
- json = String.join(" ", IOUtils.readLines(is, Charset.defaultCharset()));
- LOGGER.debug(json);
-
- Post ser = mapper.readValue(json, Post.class);
-
- String de = mapper.writeValueAsString(ser);
-
- LOGGER.debug(de);
-
- Post serde = mapper.readValue(de, Post.class);
-
- assertEquals(ser, serde);
-
- LOGGER.debug(mapper.writeValueAsString(serde));
-
- Activity activity = new Activity();
- FacebookActivityUtil.updateActivity(ser, activity);
-
- assertNotNull(activity);
- assertNotNull(activity.getActor().getId());
- assertNotNull(activity.getActor().getDisplayName());
- assertNotNull(activity.getId());
- assert(activity.getVerb().equals("post"));
- assertNotNull(activity.getObject());
- assertNotNull(activity.getUpdated());
- assertNotNull(activity.getPublished());
- assertEquals(activity.getProvider().getId(), "id:providers:facebook");
- assertEquals(activity.getProvider().getDisplayName(), "Facebook");
- assertEquals(activity.getLinks().size(), 1);
- assertNotNull(activity.getAdditionalProperties().get("facebook"));
-
- } catch( Exception e ) {
- LOGGER.error("Exception: ", e);
- Assert.fail();
- }
- }
-}
diff --git a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/providers/TestFacebookProvider.java b/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/providers/TestFacebookProvider.java
deleted file mode 100644
index 929366d..0000000
--- a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/providers/TestFacebookProvider.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.streams.facebook.test.providers;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.facebook.FacebookConfiguration;
-import org.apache.streams.facebook.IdConfig;
-import org.apache.streams.facebook.provider.FacebookDataCollector;
-import org.apache.streams.facebook.provider.FacebookProvider;
-
-import org.junit.Test;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.CyclicBarrier;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Unit Tests For {@link org.apache.streams.facebook.provider.FacebookProvider}
- */
-public class TestFacebookProvider {
-
- @Test
- public void testFacebookProvider() throws Exception {
- //Test that streams starts and shut downs.
- final CyclicBarrier barrier = new CyclicBarrier(2);
- FacebookProvider provider = new FacebookProvider(new FacebookConfiguration()) {
- @Override
- protected FacebookDataCollector getDataCollector() {
- return new TestFacebookDataCollector(barrier, super.configuration, super.datums);
- }
- };
- provider.prepare(null);
- provider.startStream();
- assertTrue(provider.isRunning());
- barrier.await();
- assertTrue(provider.isRunning());
- assertEquals(5, provider.readCurrent().size());
- barrier.await();
- assertEquals(0, provider.readCurrent().size());
- assertFalse(provider.isRunning());
- provider.cleanUp();
- }
-
- private class TestFacebookDataCollector extends FacebookDataCollector {
-
- private CyclicBarrier barrier;
- private BlockingQueue<StreamsDatum> queue;
-
- public TestFacebookDataCollector(CyclicBarrier barrier, FacebookConfiguration config, BlockingQueue<StreamsDatum> queue) {
- super(config, queue);
- this.barrier = barrier;
- this.queue = queue;
-
- }
-
- @Override
- protected void getData(IdConfig id) throws Exception {
-
- }
-
- @Override
- public void run() {
- try {
- for(int i=0; i < 5; ++i) {
- super.outputData(new Integer(i), ""+i);
- }
- this.barrier.await();
- super.isComplete.set(true);
- this.barrier.await();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- } catch (BrokenBarrierException bbe) {
- fail();
- }
- }
- }
-
-
-
-}
diff --git a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/providers/page/FacebookPageProviderIT.java b/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/providers/page/FacebookPageProviderIT.java
deleted file mode 100644
index 20a08a7..0000000
--- a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/providers/page/FacebookPageProviderIT.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.test.providers.page;
-
-import org.apache.streams.facebook.provider.page.FacebookPageProvider;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-import static org.testng.Assert.assertTrue;
-
-public class FacebookPageProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookPageProviderIT.class);
-
- @Test
- public void testFacebookPageProvider() throws Exception {
-
- String configfile = "./target/test-classes/FacebookPageProviderIT.conf";
- String outfile = "./target/test-classes/FacebookPageProviderIT.stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- FacebookPageProvider.main(args);
- } catch( Exception e ) {
- LOGGER.error("Test Exception!", e);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- FacebookPageProvider.main(new String[]{configfile, outfile});
-
- File out = new File(outfile);
- assertTrue (out.exists());
- assertTrue (out.canRead());
- assertTrue (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while(outCounter.readLine() != null) {}
-
- assertTrue (outCounter.getLineNumber() >= 1);
-
- }
-}
-
-
diff --git a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/providers/pagefeed/FacebookPageFeedProviderIT.java b/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/providers/pagefeed/FacebookPageFeedProviderIT.java
deleted file mode 100644
index aeb01dd..0000000
--- a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/providers/pagefeed/FacebookPageFeedProviderIT.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.facebook.test.providers.pagefeed;
-
-import org.apache.streams.facebook.provider.pagefeed.FacebookPageFeedProvider;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-import static org.testng.Assert.assertTrue;
-
-public class FacebookPageFeedProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FacebookPageFeedProviderIT.class);
-
- @Test
- public void testFacebookPageFeedProvider() throws Exception {
-
- String configfile = "./target/test-classes/FacebookPageFeedProviderIT.conf";
- String outfile = "./target/test-classes/FacebookPageFeedProviderIT.stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- FacebookPageFeedProvider.main(args);
- } catch( Exception e ) {
- LOGGER.error("Test Exception!", e);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- File out = new File(outfile);
- assertTrue (out.exists());
- assertTrue (out.canRead());
- assertTrue (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while(outCounter.readLine() != null) {}
-
- assertTrue (outCounter.getLineNumber() >= 1000);
-
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/test/resources/FacebookPageFeedProviderIT.conf b/streams-contrib/streams-provider-facebook/src/test/resources/FacebookPageFeedProviderIT.conf
deleted file mode 100644
index 5b7b32e..0000000
--- a/streams-contrib/streams-provider-facebook/src/test/resources/FacebookPageFeedProviderIT.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-facebook {
- ids = [
- {
- #"id": "facebook"
- "id": "20531316728"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-facebook/src/test/resources/FacebookPageProviderIT.conf b/streams-contrib/streams-provider-facebook/src/test/resources/FacebookPageProviderIT.conf
deleted file mode 100644
index 5b7b32e..0000000
--- a/streams-contrib/streams-provider-facebook/src/test/resources/FacebookPageProviderIT.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-facebook {
- ids = [
- {
- #"id": "facebook"
- "id": "20531316728"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/google-gmail/README.md b/streams-contrib/streams-provider-google/google-gmail/README.md
deleted file mode 100644
index fe2d138..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-google-gmail
-=====================
-
-GMail Provider
-
-Example configuration:
-
- gmail {
- username = ""
- password = ""
- }
-
-
-
diff --git a/streams-contrib/streams-provider-google/google-gmail/pom.xml b/streams-contrib/streams-provider-google/google-gmail/pom.xml
deleted file mode 100644
index 5fc0394..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/pom.xml
+++ /dev/null
@@ -1,158 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-provider-google</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-provider-google-gmail</artifactId>
- <name>${project.artifactId}</name>
-
- <description>GMail Provider</description>
-
- <repositories>
- <repository>
- <id>sonatype</id>
- <name>typesafe</name>
- <url>https://oss.sonatype.org/content/repositories/snapshots</url>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- </repositories>
-
- <dependencies>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path-assert</artifactId>
- </dependency>
- <dependency>
- <groupId>com.googlecode.gmail4j</groupId>
- <artifactId>gmail4j</artifactId>
- <version>0.4</version>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.google.gmail.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailImapProviderTask.java b/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailImapProviderTask.java
deleted file mode 100644
index bfebe33..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailImapProviderTask.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gmail.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.util.ComponentUtils;
-
-import com.googlecode.gmail4j.GmailMessage;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * GMailImapProviderTask collects Gmail via IMAP driver.
- */
-public class GMailImapProviderTask implements Runnable {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(GMailImapProviderTask.class);
-
- private GMailProvider provider;
-
- public GMailImapProviderTask(GMailProvider provider) {
- this.provider = provider;
- }
-
- @Override
- public void run() {
-
- final List<GmailMessage> messages = this.provider.imapClient.getUnreadMessages();
-
- for (GmailMessage message : messages) {
-
- Activity activity;
- GMailMessageActivitySerializer serializer = new GMailMessageActivitySerializer( this.provider );
- activity = serializer.deserialize(message);
- StreamsDatum entry = new StreamsDatum(activity);
- ComponentUtils.offerUntilSuccess(entry, this.provider.providerQueue);
-
- }
-
- }
-
-
-}
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailMessageActivitySerializer.java b/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailMessageActivitySerializer.java
deleted file mode 100644
index 68b8e35..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailMessageActivitySerializer.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gmail.provider;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Generator;
-import org.apache.streams.pojo.json.Icon;
-import org.apache.streams.pojo.json.Provider;
-
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.googlecode.gmail4j.GmailException;
-import com.googlecode.gmail4j.GmailMessage;
-import com.googlecode.gmail4j.javamail.JavaMailGmailMessage;
-import com.sun.mail.imap.IMAPFolder;
-import com.sun.mail.imap.IMAPMessage;
-import com.sun.mail.imap.IMAPSSLStore;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import javax.mail.internet.MimeMultipart;
-
-/**
- * GMailMessageActivitySerializer converts a GMail message to Activity.
- */
-public class GMailMessageActivitySerializer implements ActivitySerializer<GmailMessage> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GMailMessageActivitySerializer.class);
-
- private GMailProvider provider;
-
- public GMailMessageActivitySerializer(GMailProvider provider) {
-
- this.provider = provider;
-
- ObjectMapper mapper = new ObjectMapper();
- mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, Boolean.FALSE);
-
- mapper.addMixInAnnotations(IMAPSSLStore.class, MessageMixIn.class);
- mapper.addMixInAnnotations(IMAPFolder.class, MessageMixIn.class);
- mapper.addMixInAnnotations(IMAPMessage.class, MessageMixIn.class);
- mapper.addMixInAnnotations(MimeMultipart.class, MessageMixIn.class);
- mapper.addMixInAnnotations(JavaMailGmailMessage.class, MessageMixIn.class);
-
- }
-
- public GMailMessageActivitySerializer() {
- }
-
- public static Generator buildGenerator(ObjectNode event) {
- return null;
- }
-
- public static Icon getIcon(ObjectNode event) {
- return null;
- }
-
- public static Provider buildProvider(ObjectNode event) {
- Provider provider = new Provider();
- provider.setId("id:providers:gmail");
- return provider;
- }
-
- public static List<Object> getLinks(ObjectNode event) {
- return null;
- }
-
- public static String getUrls(ObjectNode event) {
- return null;
- }
-
- public static void addGMailExtension(Activity activity, GmailMessage gmailMessage) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- extensions.put("gmail", gmailMessage);
- }
-
- public static String formatId(String... idparts) {
- return String.join(":",
- Stream.concat(Arrays.stream(new String[] {"id:googleplus"}), Arrays.stream(idparts)).collect(Collectors.toList()));
- }
-
- @Override
- public String serializationFormat() {
- return "gmail.v1";
- }
-
- @Override
- public GmailMessage serialize(Activity activity) {
- return null;
- }
-
- @Override
- public Activity deserialize(GmailMessage gmailMessage) {
-
- Activity activity = new Activity();
- activity.setId(formatId(this.provider.getConfig().getUserName(), String.valueOf(gmailMessage.getMessageNumber())));
- activity.setPublished(new DateTime(gmailMessage.getSendDate()));
- Provider provider = new Provider();
- provider.setId("http://gmail.com");
- provider.setDisplayName("GMail");
- activity.setProvider(provider);
- ActivityObject actor = new ActivityObject();
- actor.setId(gmailMessage.getFrom().getEmail());
- actor.setDisplayName(gmailMessage.getFrom().getName());
- activity.setActor(actor);
- activity.setVerb("email");
- ActivityObject object = new ActivityObject();
- try {
- object.setId(gmailMessage.getTo().get(0).getEmail());
- object.setDisplayName(gmailMessage.getTo().get(0).getName());
- } catch (GmailException e) {
- LOGGER.warn(e.getMessage());
- }
- activity.setTitle(gmailMessage.getSubject());
- try {
- activity.setContent(gmailMessage.getContentText());
- } catch (GmailException e) {
- LOGGER.warn(e.getMessage());
- }
- activity.setObject(object);
- return activity;
- }
-
- @Override
- public List<Activity> deserializeAll(List<GmailMessage> serializedList) {
- throw new NotImplementedException("Not currently implemented");
- }
-
- public Activity convert(ObjectNode event) {
- return null;
- }
-
- interface MessageMixIn {
- @JsonManagedReference
- @JsonIgnore
- IMAPSSLStore getDefaultFolder(); // we don't need it!
-
- @JsonManagedReference
- @JsonIgnore
- IMAPSSLStore getPersonalNamespaces(); // we don't need it!
-
- @JsonManagedReference
- @JsonIgnore
- IMAPFolder getStore(); // we don't need it!
-
-// @JsonManagedReference
-// @JsonIgnore
-// @JsonBackReference
- //IMAPFolder getParent(); // we don't need it!
- @JsonManagedReference
- @JsonIgnore
- @JsonBackReference
- IMAPMessage getFolder(); // we don't need it!
-
- @JsonManagedReference
- @JsonIgnore
- @JsonProperty("parent")
- @JsonBackReference
- MimeMultipart getParent();
- }
-
-}
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailProvider.java b/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailProvider.java
deleted file mode 100644
index 06c73fc..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/main/java/org/apache/streams/gmail/provider/GMailProvider.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gmail.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.gmail.GMailConfiguration;
-
-import com.googlecode.gmail4j.GmailClient;
-import com.googlecode.gmail4j.GmailConnection;
-import com.googlecode.gmail4j.http.HttpGmailConnection;
-import com.googlecode.gmail4j.javamail.ImapGmailClient;
-import com.googlecode.gmail4j.javamail.ImapGmailConnection;
-import com.googlecode.gmail4j.rss.RssGmailClient;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * GMailProvider collects messages from GMail.
- */
-public class GMailProvider implements StreamsProvider, Serializable {
-
- public final static String STREAMS_ID = "GMailProvider";
-
- private final static Logger LOGGER = LoggerFactory.getLogger(GMailProvider.class);
- protected BlockingQueue inQueue = new LinkedBlockingQueue<>(10000);
- protected volatile Queue<StreamsDatum> providerQueue = new ConcurrentLinkedQueue<>();
- protected Future task;
- protected GmailClient rssClient;
- protected ImapGmailClient imapClient;
- protected DatumStatusCounter countersTotal = new DatumStatusCounter();
- protected DatumStatusCounter countersCurrent = new DatumStatusCounter();
- private GMailConfiguration config;
- private Class klass;
- private ExecutorService executor;
-
- public GMailProvider() {
- this.config = new ComponentConfigurator<>(GMailConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("gmail"));
- }
-
- public GMailProvider(GMailConfiguration config) {
- this.config = config;
- }
-
- public GMailProvider(Class klass) {
- this.config = new ComponentConfigurator<>(GMailConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("gmail"));
- this.klass = klass;
- }
-
- public GMailProvider(GMailConfiguration config, Class klass) {
- this.config = config;
- this.klass = klass;
- }
-
- private static ExecutorService newFixedThreadPoolWithQueueSize(int nThreads, int queueSize) {
- return new ThreadPoolExecutor(nThreads, nThreads,
- 5000L, TimeUnit.MILLISECONDS,
- new ArrayBlockingQueue<>(queueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
- }
-
- public GMailConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(GMailConfiguration config) {
- this.config = config;
- }
-
- public BlockingQueue<Object> getInQueue() {
- return inQueue;
- }
-
- @Override
- public String getId() {
- return "GMailProvider";
- }
-
- @Override
- public void startStream() {
-
- task = executor.submit(new GMailImapProviderTask(this));
-
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- synchronized ( GMailProvider.class ) {
- current = new StreamsResultSet(new ConcurrentLinkedQueue<>(providerQueue));
- current.setCounter(new DatumStatusCounter());
- current.getCounter().add(countersCurrent);
- countersTotal.add(countersCurrent);
- countersCurrent = new DatumStatusCounter();
- providerQueue.clear();
- }
-
- return current;
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return !task.isDone() && !task.isCancelled();
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- Objects.requireNonNull(this.klass);
- Objects.requireNonNull(config.getUserName());
- Objects.requireNonNull(config.getPassword());
-
- rssClient = new RssGmailClient();
- GmailConnection rssConnection = new HttpGmailConnection(config.getUserName(), config.getPassword().toCharArray());
- rssClient.setConnection(rssConnection);
-
- imapClient = new ImapGmailClient();
- GmailConnection imapConnection = new ImapGmailConnection();
- imapConnection.setLoginCredentials(config.getUserName(), config.getPassword().toCharArray());
- imapClient.setConnection(imapConnection);
-
- executor = Executors.newSingleThreadExecutor();
-
- startStream();
- }
-
- @Override
- public void cleanUp() {
- try {
- executor.awaitTermination(5, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/jsonschema/org/apache/streams/google/gmail/GMailConfiguration.json b/streams-contrib/streams-provider-google/google-gmail/src/main/jsonschema/org/apache/streams/google/gmail/GMailConfiguration.json
deleted file mode 100644
index 614b290..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/main/jsonschema/org/apache/streams/google/gmail/GMailConfiguration.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.gmail.GMailConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "userName": {
- "type": "string"
- },
- "password": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/resources/components.dot b/streams-contrib/streams-provider-google/google-gmail/src/main/resources/components.dot
deleted file mode 100644
index e650288..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/main/resources/components.dot
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- subgraph cluster_generators {
- label="generators";
- style="dotted";
- generators_gmail [label="https://www.googleapis.com/gmail/v1"]
- }
-
- subgraph cluster_providers {
- label="providers";
- providers_gmail [label="GmailProvider"]
- }
-
- persisters [label="persisters", shape="cylinder"];
-
- generators_gmail -> providers_gmail
-
- providers_gmail -> persisters
-}
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/main/resources/gmail.conf b/streams-contrib/streams-provider-google/google-gmail/src/main/resources/gmail.conf
deleted file mode 100644
index e99e964..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/main/resources/gmail.conf
+++ /dev/null
@@ -1,6 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-gmail {
- username = ""
- password = ""
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/site/markdown/index.md b/streams-contrib/streams-provider-google/google-gmail/src/site/markdown/index.md
deleted file mode 100644
index 7f935a1..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/site/markdown/index.md
+++ /dev/null
@@ -1,22 +0,0 @@
-org.apache.streams:google-gmail
-===============================
-
-google-gmail contains providers, conversions, and utility classes for activity exchange with gmail.com
-
-## Configuration
-
-| Schema | JavaDoc | Example Configuration(s) |
-|--------|---------|--------------------------|
-| [GMailConfiguration.json](com/google/gmail/GMailConfiguration.json "GMailConfiguration.json") | [GMailConfiguration.html](apidocs/com/google/gmail/GMailConfiguration.html "GMailConfiguration.html") | [gmail.conf](gmail.conf "gmail.conf") |
-
-## Components
-
-
-
-| Class | Class JavaDoc |
-|-------|---------------|
-| GMailProvider | [GMailProvider.html](apidocs/com/google/gmail/provider/GMailProvider.html "GMailProvider.html") | [TwitterUserInformationConfiguration.json](com/twitter/TwitterUserInformationConfiguration.json "TwitterUserInformationConfiguration.json") |
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-provider-google/google-gmail/src/test/java/org/apache/streams/gmail/test/GMailMessageSerDeTest.java b/streams-contrib/streams-provider-google/google-gmail/src/test/java/org/apache/streams/gmail/test/GMailMessageSerDeTest.java
deleted file mode 100644
index 8d3da9e..0000000
--- a/streams-contrib/streams-provider-google/google-gmail/src/test/java/org/apache/streams/gmail/test/GMailMessageSerDeTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gmail.test;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests conversion of gmail inputs to Activity
- */
-@Ignore("ignore until test resources are available.")
-public class GMailMessageSerDeTest {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(GMailMessageSerDeTest.class);
-
- private ObjectMapper mapper = new ObjectMapper();
-
- @Ignore
- @Test
- public void Tests()
- {
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = GMailMessageSerDeTest.class.getResourceAsStream("/datasift_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- LOGGER.debug(line);
-
- // implement
- }
- } catch( Exception e ) {
- e.printStackTrace();
- Assert.fail();
- }
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/README.md b/streams-contrib/streams-provider-google/google-gplus/README.md
deleted file mode 100644
index b2175a0..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/README.md
+++ /dev/null
@@ -1,22 +0,0 @@
-google-gplus
-=====================
-
-Google+ Provider
-
-Example configuration:
-
- gplus {
- googlePlusUsers = [
- {
- userId = ""
- }
- ]
- oauth {
- appName = ""
- pathToP12KeyFile = "Absolute Path to key file"
- serviceAccountEmailAddress = ""
- }
- }
-
-
-
diff --git a/streams-contrib/streams-provider-google/google-gplus/pom.xml b/streams-contrib/streams-provider-google/google-gplus/pom.xml
deleted file mode 100644
index 49d44ac..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/pom.xml
+++ /dev/null
@@ -1,238 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-provider-google</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-provider-google-gplus</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Google+ Provider</description>
-
- <properties>
- <google.client.version>1.22.0</google.client.version>
- <gplus.client.version>v1-rev457-1.22.0</gplus.client.version>
- <skipITs>true</skipITs>
- <testDataBaseURl>http://streams.peoplepattern.com.s3.amazonaws.com/test-data/</testDataBaseURl>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
- <repositories>
- <repository>
- <id>typesafe</id>
- <name>typesafe</name>
- <url>http://repo.typesafe.com/typesafe/repo</url>
- </repository>
- </repositories>
-
- <dependencies>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path-assert</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.apis</groupId>
- <artifactId>google-api-services-plus</artifactId>
- <version>${gplus.client.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.api-client</groupId>
- <artifactId>google-api-client</artifactId>
- <version>${google.client.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.http-client</groupId>
- <artifactId>google-http-client-jackson2</artifactId>
- <version>${google.client.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- <version>6.9.13.6</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito</artifactId>
- <version>1.5.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>1.5.5</version>
- <exclusions>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-reflect</artifactId>
- </exclusion>
- </exclusions>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.google.gplus.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <version>1.2.1</version>
- <executions>
- <execution>
- <id>download-it-data</id>
- <phase>pre-integration-test</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
- <url>${testDataBaseURl}/${project.artifactId}.zip</url>
- <unpack>true</unpack>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- <!--<md5>df65b5642f33676313ebe4d5b69a3fff</md5>-->
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <configuration>
- <skipTests>${skipITs}</skipTests>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusCommentProcessor.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusCommentProcessor.java
deleted file mode 100644
index 1128a93..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusCommentProcessor.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.streams.pojo.json.Activity;
-
-import com.google.api.services.plus.model.Comment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * GooglePlusCommentProcessor collects comments about a google plus activity.
- */
-public class GooglePlusCommentProcessor implements StreamsProcessor {
-
- private static final String STREAMS_ID = "GooglePlusCommentProcessor";
- private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentProcessor.class);
- private int count;
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- StreamsDatum result = null;
-
- try {
- Object item = entry.getDocument();
- LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
-
- //Get G+ activity ID from our own activity ID
- if (item instanceof Activity) {
- Activity activity = (Activity) item;
- String activityId = getGPlusID(activity.getId());
-
- //Call Google Plus API to get list of comments for this activity ID
- /* TODO: FILL ME OUT WITH THE API CALL **/
- List<Comment> comments = new ArrayList<>();
-
- GooglePlusActivityUtil.updateActivity(comments, activity);
- result = new StreamsDatum(activity);
- }
- } catch (Exception ex) {
- ex.printStackTrace();
- LOGGER.error("Exception while converting Comment to Activity: {}", ex.getMessage());
- }
-
- if ( result != null ) {
- return Stream.of(result).collect(Collectors.toList());
- } else {
- return new ArrayList<>();
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
- count = 0;
- }
-
- @Override
- public void cleanUp() {
-
- }
-
- private String getGPlusID(String activityId) {
- String[] activityParts = activityId.split(":");
- return (activityParts.length > 0) ? activityParts[activityParts.length - 1] : "";
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusTypeConverter.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusTypeConverter.java
deleted file mode 100644
index ff11b45..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/processor/GooglePlusTypeConverter.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.gplus.serializer.util.GPlusActivityDeserializer;
-import org.apache.streams.gplus.serializer.util.GPlusEventClassifier;
-import org.apache.streams.gplus.serializer.util.GPlusPersonDeserializer;
-import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.plus.model.Person;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Queue;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * GooglePlusTypeConverter is a StreamsProcessor that converts gplus activities to activitystreams activities.
- */
-public class GooglePlusTypeConverter implements StreamsProcessor {
-
- public static final String STREAMS_ID = "GooglePlusTypeConverter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusTypeConverter.class);
- private StreamsJacksonMapper mapper;
- private Queue<Person> inQueue;
- private Queue<StreamsDatum> outQueue;
- private int count = 0;
-
- public GooglePlusTypeConverter() {}
-
- public Queue<StreamsDatum> getProcessorOutputQueue() {
- return outQueue;
- }
-
- public void setProcessorInputQueue(Queue<Person> inputQueue) {
- inQueue = inputQueue;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- StreamsDatum result = null;
-
- try {
- Object item = entry.getDocument();
-
- LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
- Activity activity = null;
-
- if (item instanceof String) {
- item = deserializeItem(item);
- }
-
- if (item instanceof Person) {
- activity = new Activity();
- GooglePlusActivityUtil.updateActivity((Person) item, activity);
- } else if (item instanceof com.google.api.services.plus.model.Activity) {
- activity = new Activity();
- GooglePlusActivityUtil.updateActivity((com.google.api.services.plus.model.Activity) item, activity);
- }
-
- if (activity != null) {
- result = new StreamsDatum(activity);
- count++;
- }
- } catch (Exception ex) {
- ex.printStackTrace();
- LOGGER.error("Exception while converting Person to Activity: {}", ex.getMessage());
- }
-
- if (result != null) {
- return Stream.of(result).collect(Collectors.toList());
- } else {
- return new ArrayList<>();
- }
- }
-
- private Object deserializeItem(Object item) {
- try {
- Class klass = GPlusEventClassifier.detectClass((String) item);
-
- if (klass.equals(Person.class)) {
- item = mapper.readValue((String) item, Person.class);
- } else if (klass.equals(com.google.api.services.plus.model.Activity.class)) {
- item = mapper.readValue((String) item, com.google.api.services.plus.model.Activity.class);
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserializeItem: {}", ex);
- }
-
- return item;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- mapper = StreamsJacksonMapper.getInstance();
-
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
- mapper.registerModule(simpleModule);
-
- simpleModule = new SimpleModule();
- simpleModule.addDeserializer(com.google.api.services.plus.model.Activity.class, new GPlusActivityDeserializer());
- mapper.registerModule(simpleModule);
- }
-
- @Override
- public void cleanUp() {
- //No-op
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/AbstractGPlusProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/AbstractGPlusProvider.java
deleted file mode 100644
index 7c07c70..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/AbstractGPlusProvider.java
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.google.gplus.GPlusConfiguration;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.ComponentUtils;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
-
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.http.javanet.NetHttpTransport;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.plus.Plus;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.gson.Gson;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.math.BigInteger;
-import java.security.GeneralSecurityException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Provider that creates a GPlus client and will run task that queue data to an outing queue.
- */
-public abstract class AbstractGPlusProvider implements StreamsProvider {
-
- public static final String STREAMS_ID = "AbstractGPlusProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(AbstractGPlusProvider.class);
- private static final Set<String> SCOPE = new HashSet<String>() {
- {
- add("https://www.googleapis.com/auth/plus.login");
- }
- };
- private static final int MAX_BATCH_SIZE = 1000;
-
- private static final HttpTransport TRANSPORT = new NetHttpTransport();
- private static final JacksonFactory JSON_FACTORY = new JacksonFactory();
- private static final Gson GSON = new Gson();
-
- private GPlusConfiguration config;
-
- List<ListenableFuture<Object>> futures = new ArrayList<>();
-
- private ListeningExecutorService executor;
-
- private BlockingQueue<StreamsDatum> datumQueue;
- private AtomicBoolean isComplete;
- private boolean previousPullWasEmpty;
-
- protected GoogleCredential credential;
- protected Plus plus;
-
- public AbstractGPlusProvider() {
- this.config = new ComponentConfigurator<>(GPlusConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("gplus"));
- }
-
- public AbstractGPlusProvider(GPlusConfiguration config) {
- this.config = config;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- Objects.requireNonNull(config.getOauth().getPathToP12KeyFile());
- Objects.requireNonNull(config.getOauth().getAppName());
- Objects.requireNonNull(config.getOauth().getServiceAccountEmailAddress());
-
- try {
- this.plus = createPlusClient();
- } catch (IOException | GeneralSecurityException ex) {
- LOGGER.error("Failed to created oauth for GPlus : {}", ex);
- throw new RuntimeException(ex);
- }
- // GPlus rate limits you to 5 calls per second, so there is not a need to execute more than one
- // collector unless you have multiple oauth tokens
- //TODO make this configurable based on the number of oauth tokens
- this.executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
- this.datumQueue = new LinkedBlockingQueue<>(1000);
- this.isComplete = new AtomicBoolean(false);
- this.previousPullWasEmpty = false;
- }
-
- @Override
- public void startStream() {
-
- BackOffStrategy backOffStrategy = new ExponentialBackOffStrategy(2);
- for (UserInfo user : this.config.getGooglePlusUsers()) {
- if (this.config.getDefaultAfterDate() != null && user.getAfterDate() == null) {
- user.setAfterDate(this.config.getDefaultAfterDate());
- }
- if (this.config.getDefaultBeforeDate() != null && user.getBeforeDate() == null) {
- user.setBeforeDate(this.config.getDefaultBeforeDate());
- }
- this.executor.submit(getDataCollector(backOffStrategy, this.datumQueue, this.plus, user));
- }
- this.executor.shutdown();
- }
-
- protected abstract Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo);
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- BlockingQueue<StreamsDatum> batch = new LinkedBlockingQueue<>();
- int batchCount = 0;
- while (!this.datumQueue.isEmpty() && batchCount < MAX_BATCH_SIZE) {
- StreamsDatum datum = ComponentUtils.pollWhileNotEmpty(this.datumQueue);
- if (datum != null) {
- ++batchCount;
- ComponentUtils.offerUntilSuccess(datum, batch);
- }
- }
- boolean pullIsEmpty = batch.isEmpty() && this.datumQueue.isEmpty() && this.executor.isTerminated();
- this.isComplete.set(this.previousPullWasEmpty && pullIsEmpty);
- this.previousPullWasEmpty = pullIsEmpty;
- return new StreamsResultSet(batch);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @VisibleForTesting
- protected Plus createPlusClient() throws IOException, GeneralSecurityException {
- credential = new GoogleCredential.Builder()
- .setJsonFactory(JSON_FACTORY)
- .setTransport(TRANSPORT)
- .setServiceAccountScopes(SCOPE)
- .setServiceAccountId(this.config.getOauth().getServiceAccountEmailAddress())
- .setServiceAccountPrivateKeyFromP12File(new File(this.config.getOauth().getPathToP12KeyFile()))
- .build();
- return new Plus.Builder(TRANSPORT,JSON_FACTORY, credential).setApplicationName(this.config.getOauth().getAppName()).build();
- }
-
- @Override
- public void cleanUp() {
- ComponentUtils.shutdownExecutor(this.executor, 10, 10);
- this.executor = null;
- }
-
- public GPlusConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(GPlusConfiguration config) {
- this.config = config;
- }
-
- /**
- * Set and overwrite the default before date that was read from the configuration file.
- * @param defaultBeforeDate defaultBeforeDate
- */
- public void setDefaultBeforeDate(DateTime defaultBeforeDate) {
- this.config.setDefaultBeforeDate(defaultBeforeDate);
- }
-
- /**
- * Set and overwrite the default after date that was read from teh configuration file.
- * @param defaultAfterDate defaultAfterDate
- */
- public void setDefaultAfterDate(DateTime defaultAfterDate) {
- this.config.setDefaultAfterDate(defaultAfterDate);
- }
-
- /**
- * Sets and overwrite the user info from the configuaration file. Uses the defaults before and after dates.
- * @param userIds userIds
- */
- public void setUserInfoWithDefaultDates(Set<String> userIds) {
- List<UserInfo> gplusUsers = new LinkedList<>();
- for (String userId : userIds) {
- UserInfo user = new UserInfo();
- user.setUserId(userId);
- user.setAfterDate(this.config.getDefaultAfterDate());
- user.setBeforeDate(this.config.getDefaultBeforeDate());
- gplusUsers.add(user);
- }
- this.config.setGooglePlusUsers(gplusUsers);
- }
-
- /**
- * Set and overwrite user into from the configuration file. Only sets after date.
- * @param usersAndAfterDates usersAndAfterDates
- */
- public void setUserInfoWithAfterDate(Map<String, DateTime> usersAndAfterDates) {
- List<UserInfo> gplusUsers = new LinkedList<>();
- for (String userId : usersAndAfterDates.keySet()) {
- UserInfo user = new UserInfo();
- user.setUserId(userId);
- user.setAfterDate(usersAndAfterDates.get(userId));
- gplusUsers.add(user);
- }
- this.config.setGooglePlusUsers(gplusUsers);
- }
-
- @Override
- public boolean isRunning() {
- if (datumQueue.isEmpty() && executor.isTerminated() && Futures.allAsList(futures).isDone()) {
- LOGGER.info("Completed");
- isComplete.set(true);
- LOGGER.info("Exiting");
- }
- return !isComplete.get();
- }
-
-
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusActivitySerializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusActivitySerializer.java
deleted file mode 100644
index 74b0d8a..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusActivitySerializer.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.provider;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.pojo.json.Activity;
-
-import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.commons.lang.NotImplementedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * GPlusActivitySerializer converts gplus activities to as1 activities.
- */
-public class GPlusActivitySerializer implements ActivitySerializer<com.google.api.services.plus.model.Activity> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusActivitySerializer.class);
-
- AbstractGPlusProvider provider;
-
- public GPlusActivitySerializer(AbstractGPlusProvider provider) {
-
- this.provider = provider;
- }
-
- public GPlusActivitySerializer() {
- }
-
- @Override
- public String serializationFormat() {
- return "gplus.v1";
- }
-
- @Override
- public com.google.api.services.plus.model.Activity serialize(Activity deserialized) {
- throw new NotImplementedException("Not currently implemented");
- }
-
- @Override
- public Activity deserialize(com.google.api.services.plus.model.Activity gplusActivity) {
- Activity activity = new Activity();
-
- GooglePlusActivityUtil.updateActivity(gplusActivity, activity);
- return activity;
- }
-
- @Override
- public List<Activity> deserializeAll(List<com.google.api.services.plus.model.Activity> serializedList) {
- throw new NotImplementedException("Not currently implemented");
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusDataCollector.java
deleted file mode 100644
index 3726bc5..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusDataCollector.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.provider;
-
-import org.apache.streams.util.api.requests.backoff.BackOffException;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * GPlusDataCollector collects GPlus Data on behalf of providers.
- */
-public abstract class GPlusDataCollector implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusDataCollector.class);
-
- /**
- * Looks at the status code of the exception. If the code indicates that the request should be retried,
- * it executes the back off strategy and returns true.
- * @param gjre GoogleJsonResponseException
- * @param backOff BackOffStrategy
- * @return returns true if the error code of the exception indicates the request should be retried.
- */
- public boolean backoffAndIdentifyIfRetry(GoogleJsonResponseException gjre, BackOffStrategy backOff) throws BackOffException {
- boolean tryAgain = false;
- switch (gjre.getStatusCode()) {
- case 400 :
- LOGGER.warn("Bad Request : {}", gjre);
- break;
- case 401 :
- LOGGER.warn("Invalid Credentials : {}", gjre);
- break;
- case 403 :
- LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
- backOff.backOff();
- tryAgain = true;
- break;
- case 503 :
- LOGGER.warn("Google Backend Service Error : {}", gjre);
- break;
- default:
- LOGGER.warn("Google Service returned error : {}", gjre);
- tryAgain = true;
- backOff.backOff();
- break;
- }
- return tryAgain;
- }
-
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityCollector.java
deleted file mode 100644
index fce108d..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityCollector.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.services.plus.Plus;
-import com.google.api.services.plus.model.Activity;
-import com.google.api.services.plus.model.ActivityFeed;
-import org.apache.streams.gplus.serializer.util.GPlusActivityDeserializer;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.BlockingQueue;
-
-/**
- * Collects the public activities of a GPlus user. Has ability to filter by date ranges.
- */
-public class GPlusUserActivityCollector extends GPlusDataCollector {
-
- /**
- * Key for all public activities
- * https://developers.google.com/+/api/latest/activities/list
- */
- private static final String PUBLIC_COLLECTION = "public";
- /**
- * Max results allowed per request
- * https://developers.google.com/+/api/latest/activities/list
- */
- private static final long MAX_RESULTS = 100;
- private static final int MAX_ATTEMPTS = 5;
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserActivityCollector.class);
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- static { //set up mapper for Google Activity Object
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Activity.class, new GPlusActivityDeserializer());
- simpleModule.addSerializer(
- com.google.api.client.util.DateTime.class,
- new StdSerializer<com.google.api.client.util.DateTime>(com.google.api.client.util.DateTime.class) {
- @Override
- public void serialize(
- com.google.api.client.util.DateTime dateTime,
- JsonGenerator jsonGenerator,
- SerializerProvider serializerProvider)
- throws IOException {
- jsonGenerator.writeString(dateTime.toStringRfc3339());
- }
- });
- MAPPER.registerModule(simpleModule);
- MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- private BlockingQueue<StreamsDatum> datumQueue;
- private BackOffStrategy backOff;
- private Plus plus;
- private UserInfo userInfo;
-
- /**
- * GPlusUserActivityCollector constructor.
- * @param plus Plus
- * @param datumQueue BlockingQueue<StreamsDatum>
- * @param backOff BackOffStrategy
- * @param userInfo UserInfo
- */
- public GPlusUserActivityCollector(Plus plus, BlockingQueue<StreamsDatum> datumQueue, BackOffStrategy backOff, UserInfo userInfo) {
- this.plus = plus;
- this.datumQueue = datumQueue;
- this.backOff = backOff;
- this.userInfo = userInfo;
- }
-
- @Override
- public void run() {
- collectActivityData();
- }
-
- protected void collectActivityData() {
- int item_count = 0;
- int last_count = 0;
- int page_count = 0;
- try {
- ActivityFeed feed = null;
- boolean tryAgain = false;
- int attempt = 0;
- DateTime afterDate = userInfo.getAfterDate();
- DateTime beforeDate = userInfo.getBeforeDate();
- do {
- try {
- if (feed == null) {
- feed = this.plus.activities().list(this.userInfo.getUserId(), PUBLIC_COLLECTION)
- .setMaxResults(MAX_RESULTS).execute();
- } else {
- feed = this.plus.activities().list(this.userInfo.getUserId(), PUBLIC_COLLECTION)
- .setMaxResults(MAX_RESULTS)
- .setPageToken(feed.getNextPageToken()).execute();
- }
- this.backOff.reset(); //successful pull reset api.
- last_count += feed.getItems().size();
- for (com.google.api.services.plus.model.Activity activity : feed.getItems()) {
- DateTime published = new DateTime(activity.getPublished().getValue());
- if ((afterDate == null && beforeDate == null)
- || (beforeDate == null && afterDate.isBefore(published))
- || (afterDate == null && beforeDate.isAfter(published))
- || ((afterDate != null && beforeDate != null) && (afterDate.isBefore(published) && beforeDate.isAfter(published)))) {
- String json = MAPPER.writeValueAsString(activity);
- this.datumQueue.put(new StreamsDatum(json, activity.getId()));
- item_count++;
- } else if (afterDate != null && afterDate.isAfter(published)) {
- feed.setNextPageToken(null); // do not fetch next page
- break;
- }
- }
- page_count += 1;
- } catch (GoogleJsonResponseException gjre) {
- tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOff);
- ++attempt;
- }
- }
- while ((tryAgain || (feed != null && feed.getNextPageToken() != null)) && attempt < MAX_ATTEMPTS);
- } catch (Throwable th) {
- if (th instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- th.printStackTrace();
- LOGGER.warn("Unable to pull Activities for user={} : {}",this.userInfo.getUserId(), th);
- }
-
- LOGGER.info("item_count: {} last_count: {} page_count: {} ", item_count, last_count, page_count);
-
- }
-
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityProvider.java
deleted file mode 100644
index 679660e..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserActivityProvider.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.GPlusConfiguration;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.google.api.services.plus.Plus;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.google.gson.Gson;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Retrieve recent activity from a list of accounts.
- *
- * <p/>
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * gplus.oauth.pathToP12KeyFile
- * gplus.oauth.serviceAccountEmailAddress
- * gplus.apiKey
- * gplus.googlePlusUsers
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.gplus.provider.GPlusUserActivityProvider -Dexec.args="application.conf activity.json"
- */
-public class GPlusUserActivityProvider extends AbstractGPlusProvider {
-
- private static final String STREAMS_ID = "GPlusUserActivityProvider";
-
- public GPlusUserActivityProvider() {
- super();
- }
-
- public GPlusUserActivityProvider(GPlusConfiguration config) {
- super(config);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo) {
- return new GPlusUserActivityCollector(plus, queue, strategy, userInfo);
- }
-
- /**
- * Retrieve recent activity from a list of accounts.
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- GPlusConfiguration config = new ComponentConfigurator<>(GPlusConfiguration.class).detectConfiguration(typesafe, "gplus");
- GPlusUserActivityProvider provider = new GPlusUserActivityProvider(config);
-
- Gson gson = new Gson();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- if (datum.getDocument() instanceof String) {
- json = (String) datum.getDocument();
- } else {
- json = gson.toJson(datum.getDocument());
- }
- outStream.println(json);
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataCollector.java
deleted file mode 100644
index 2cbbc8c..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataCollector.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.services.plus.Plus;
-import com.google.api.services.plus.model.Person;
-import org.apache.streams.gplus.serializer.util.GPlusPersonDeserializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.BlockingQueue;
-
-/**
- * Collects user profile information for a specific GPlus user.
- */
-public class GPlusUserDataCollector extends GPlusDataCollector {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserDataCollector.class);
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
- private static final int MAX_ATTEMPTS = 5;
-
- static { //set up Mapper for Person objects
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
- MAPPER.registerModule(simpleModule);
- MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- private BackOffStrategy backOffStrategy;
- private Plus plus;
- private BlockingQueue<StreamsDatum> datumQueue;
- private UserInfo userInfo;
-
- /**
- * GPlusUserDataCollector constructor.
- * @param plus Plus
- * @param backOffStrategy BackOffStrategy
- * @param datumQueue BlockingQueue of StreamsDatum
- * @param userInfo UserInfo
- */
- public GPlusUserDataCollector(Plus plus, BackOffStrategy backOffStrategy, BlockingQueue<StreamsDatum> datumQueue, UserInfo userInfo) {
- this.plus = plus;
- this.backOffStrategy = backOffStrategy;
- this.datumQueue = datumQueue;
- this.userInfo = userInfo;
- }
-
- protected void queueUserHistory() {
- try {
- boolean tryAgain;
- int attempts = 0;
- com.google.api.services.plus.model.Person person = null;
- do {
- try {
- person = this.plus.people().get(userInfo.getUserId()).execute();
- this.backOffStrategy.reset();
- tryAgain = person == null;
- } catch (GoogleJsonResponseException gjre) {
- tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOffStrategy);
- }
- ++attempts;
- }
- while (tryAgain && attempts < MAX_ATTEMPTS);
- String json = MAPPER.writeValueAsString(person);
- this.datumQueue.put(new StreamsDatum(json, person.getId()));
- } catch (Throwable throwable) {
- LOGGER.warn("Unable to pull user data for user={} : {}", userInfo.getUserId(), throwable);
- if (throwable instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- }
- }
-
- @Override
- public void run() {
- queueUserHistory();
- }
-
-
-
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataProvider.java
deleted file mode 100644
index 14bf472..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/provider/GPlusUserDataProvider.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.GPlusConfiguration;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.google.api.services.plus.Plus;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.google.gson.Gson;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.hamcrest.MatcherAssert;
-import org.hamcrest.Matchers;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-
-/**
- * Retrieve current profile status for a list of accounts.
- *
- * <p/>
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * gplus.oauth.pathToP12KeyFile
- * gplus.oauth.serviceAccountEmailAddress
- * gplus.apiKey
- * gplus.googlePlusUsers
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.gplus.provider.GPlusUserDataProvider -Dexec.args="application.conf profiles.json"
- */
-public class GPlusUserDataProvider extends AbstractGPlusProvider {
-
- public static final String STREAMS_ID = "GPlusUserDataProvider";
-
- public GPlusUserDataProvider() {
- super();
- }
-
- public GPlusUserDataProvider(GPlusConfiguration config) {
- super(config);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo) {
- return new GPlusUserDataCollector(plus, strategy, queue, userInfo);
- }
-
- /**
- * Retrieve current profile status for a list of accounts.
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- MatcherAssert.assertThat(args.length, Matchers.greaterThanOrEqualTo(2));
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- GPlusConfiguration config = new ComponentConfigurator<>(GPlusConfiguration.class).detectConfiguration(typesafe, "gplus");
- GPlusUserDataProvider provider = new GPlusUserDataProvider(config);
-
- Gson gson = new Gson();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- if (datum.getDocument() instanceof String) {
- json = (String) datum.getDocument();
- } else {
- json = gson.toJson(datum.getDocument());
- }
- outStream.println(json);
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusActivityDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusActivityDeserializer.java
deleted file mode 100644
index 585e511..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusActivityDeserializer.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.serializer.util;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.api.client.util.DateTime;
-import com.google.api.services.plus.model.Activity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Custom deserializer for GooglePlus' Person model.
- */
-public class GPlusActivityDeserializer extends JsonDeserializer<Activity> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusActivityDeserializer.class);
-
- /**
- * Because the GooglePlus Activity object {@link com.google.api.services.plus.model.Activity} contains complex objects
- * within its hierarchy, we have to use a custom deserializer
- *
- * @param jsonParser jsonParser
- * @param deserializationContext deserializationContext
- * @return The deserialized {@link com.google.api.services.plus.model.Activity} object
- * @throws IOException IOException
- * @throws JsonProcessingException JsonProcessingException
- */
- @Override
- public Activity deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
-
- JsonNode node = jsonParser.getCodec().readTree(jsonParser);
- Activity activity = new Activity();
-
- try {
- activity.setUrl(node.get("url").asText());
- activity.setEtag(node.get("etag").asText());
- activity.setTitle(node.get("title").asText());
- activity.setPublished(DateTime.parseRfc3339(node.get("published").asText()));
- activity.setUpdated(DateTime.parseRfc3339(node.get("updated").asText()));
- activity.setId(node.get("id").asText());
- activity.setVerb(node.get("verb").asText());
-
- activity.setActor(buildActor(node));
-
- activity.setObject(buildPlusObject(node));
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserialize activity object: {}", ex);
- }
-
- return activity;
- }
-
- /**
- * Given a raw JsonNode, build out the G+ {@link com.google.api.services.plus.model.Activity.Actor} object
- *
- * @param node node
- * @return {@link com.google.api.services.plus.model.Activity.Actor} object
- */
- private Activity.Actor buildActor(JsonNode node) {
- Activity.Actor actor = new Activity.Actor();
- JsonNode actorNode = node.get("actor");
-
- actor.setId(actorNode.get("id").asText());
- actor.setDisplayName(actorNode.get("displayName").asText());
- actor.setUrl(actorNode.get("url").asText());
-
- Activity.Actor.Image image = new Activity.Actor.Image();
- JsonNode imageNode = actorNode.get("image");
- image.setUrl(imageNode.get("url").asText());
-
- actor.setImage(image);
-
- return actor;
- }
-
- /**
- * Given a JsonNode, build out all aspects of the {@link com.google.api.services.plus.model.Activity.PlusObject} object
- *
- * @param node node
- * @return {@link com.google.api.services.plus.model.Activity.PlusObject} object
- */
- private Activity.PlusObject buildPlusObject(JsonNode node) {
- Activity.PlusObject object = new Activity.PlusObject();
- JsonNode objectNode = node.get("object");
- object.setObjectType(objectNode.get("objectType").asText());
- object.setContent(objectNode.get("content").asText());
- object.setUrl(objectNode.get("url").asText());
-
- Activity.PlusObject.Replies replies = new Activity.PlusObject.Replies();
- JsonNode repliesNode = objectNode.get("replies");
- replies.setTotalItems(repliesNode.get("totalItems").asLong());
- replies.setSelfLink(repliesNode.get("selfLink").asText());
- object.setReplies(replies);
-
- Activity.PlusObject.Plusoners plusoners = new Activity.PlusObject.Plusoners();
- JsonNode plusonersNode = objectNode.get("plusoners");
- plusoners.setTotalItems(plusonersNode.get("totalItems").asLong());
- plusoners.setSelfLink(plusonersNode.get("selfLink").asText());
- object.setPlusoners(plusoners);
-
- Activity.PlusObject.Resharers resharers = new Activity.PlusObject.Resharers();
- JsonNode resharersNode = objectNode.get("resharers");
- resharers.setTotalItems(resharersNode.get("totalItems").asLong());
- resharers.setSelfLink(resharersNode.get("selfLink").asText());
- object.setResharers(resharers);
-
- object.setAttachments(buildAttachments(objectNode));//attachments);
-
- return object;
- }
-
- /**
- * Given a raw JsonNode representation of an Activity's attachments, build out that
- * list of {@link com.google.api.services.plus.model.Activity.PlusObject.Attachments} objects
- *
- * @param objectNode objectNode
- * @return list of {@link com.google.api.services.plus.model.Activity.PlusObject.Attachments} objects
- */
- private List<Activity.PlusObject.Attachments> buildAttachments(JsonNode objectNode) {
- List<Activity.PlusObject.Attachments> attachments = new ArrayList<>();
- if ( objectNode.has("attachments") ) {
- for (JsonNode attachmentNode : objectNode.get("attachments")) {
- Activity.PlusObject.Attachments attachments1 = new Activity.PlusObject.Attachments();
- attachments1.setObjectType(attachmentNode.get("objectType").asText());
- if (attachmentNode.has("displayName")) {
- attachments1.setDisplayName(attachmentNode.get("displayName").asText());
- }
- if (attachmentNode.has("content")) {
- attachments1.setContent(attachmentNode.get("content").asText());
- }
- if (attachmentNode.has("url")) {
- attachments1.setUrl(attachmentNode.get("url").asText());
- }
-
- if( attachmentNode.has("image")) {
- Activity.PlusObject.Attachments.Image image1 = new Activity.PlusObject.Attachments.Image();
- JsonNode imageNode1 = attachmentNode.get("image");
- image1.setUrl(imageNode1.get("url").asText());
- attachments1.setImage(image1);
- }
-
- attachments.add(attachments1);
- }
- }
- return attachments;
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusCommentDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusCommentDeserializer.java
deleted file mode 100644
index 5d6a982..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusCommentDeserializer.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.serializer.util;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.client.util.DateTime;
-import com.google.api.services.plus.model.Comment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * GPlusCommentDeserializer converts gplus comments to as1 comments.
- */
-public class GPlusCommentDeserializer extends JsonDeserializer<Comment> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusActivityDeserializer.class);
-
- /**
- * Because the GooglePlus Comment object {@link com.google.api.services.plus.model.Comment} contains complex objects
- * within its hierarchy, we have to use a custom deserializer
- *
- * @param jsonParser jsonParser
- * @param deserializationContext deserializationContext
- * @return The deserialized {@link com.google.api.services.plus.model.Comment} object
- * @throws java.io.IOException IOException
- * @throws com.fasterxml.jackson.core.JsonProcessingException JsonProcessingException
- */
- @Override
- public Comment deserialize(JsonParser jsonParser, DeserializationContext deserializationContext)
- throws IOException {
-
- JsonNode node = jsonParser.getCodec().readTree(jsonParser);
- ObjectMapper objectMapper = StreamsJacksonMapper.getInstance();
- Comment comment = new Comment();
-
- try {
- comment.setEtag(node.get("etag").asText());
- comment.setVerb(node.get("verb").asText());
- comment.setId(node.get("id").asText());
- comment.setPublished(DateTime.parseRfc3339(node.get("published").asText()));
- comment.setUpdated(DateTime.parseRfc3339(node.get("updated").asText()));
-
- Comment.Actor actor = new Comment.Actor();
- JsonNode actorNode = node.get("actor");
- actor.setDisplayName(actorNode.get("displayName").asText());
- actor.setUrl(actorNode.get("url").asText());
-
- Comment.Actor.Image image = new Comment.Actor.Image();
- JsonNode imageNode = actorNode.get("image");
- image.setUrl(imageNode.get("url").asText());
-
- actor.setImage(image);
-
- comment.setObject(objectMapper.readValue(objectMapper.writeValueAsString(node.get("object")), Comment.PlusObject.class));
-
- comment.setSelfLink(node.get("selfLink").asText());
-
- List<Comment.InReplyTo> replies = new ArrayList<>();
- for (JsonNode reply : node.get("inReplyTo")) {
- Comment.InReplyTo irt = objectMapper.readValue(objectMapper.writeValueAsString(reply), Comment.InReplyTo.class);
- replies.add(irt);
- }
-
- comment.setInReplyTo(replies);
-
- Comment.Plusoners plusoners = new Comment.Plusoners();
- JsonNode plusonersNode = node.get("plusoners");
- plusoners.setTotalItems(plusonersNode.get("totalItems").asLong());
- comment.setPlusoners(plusoners);
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserialize activity object: {}", ex);
- }
-
- return comment;
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifier.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifier.java
deleted file mode 100644
index f6fa524..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifier.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.serializer.util;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.api.services.plus.model.Activity;
-import com.google.api.services.plus.model.Person;
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Objects;
-
-/**
- * GPlusEventClassifier classifies GPlus Events.
- */
-public class GPlusEventClassifier implements Serializable {
-
- private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- private static final String ACTIVITY_IDENTIFIER = "\"plus#activity\"";
- private static final String PERSON_IDENTIFIER = "\"plus#person\"";
-
- /**
- * Detect likely class of String json.
- * @param json String json
- * @return likely class
- */
- public static Class detectClass(String json) {
- Objects.requireNonNull(json);
- Preconditions.checkArgument(StringUtils.isNotEmpty(json));
-
- ObjectNode objectNode;
- try {
- objectNode = (ObjectNode) mapper.readTree(json);
- } catch (IOException ex) {
- ex.printStackTrace();
- return null;
- }
-
- if (objectNode.findValue("kind") != null && objectNode.get("kind").toString().equals(ACTIVITY_IDENTIFIER)) {
- return Activity.class;
- } else if (objectNode.findValue("kind") != null && objectNode.get("kind").toString().equals(PERSON_IDENTIFIER)) {
- return Person.class;
- } else {
- return ObjectNode.class;
- }
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusPersonDeserializer.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusPersonDeserializer.java
deleted file mode 100644
index 8f7ae72..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GPlusPersonDeserializer.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.serializer.util;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.services.plus.model.Person;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Custom deserializer for GooglePlus' Person model.
- */
-public class GPlusPersonDeserializer extends JsonDeserializer<Person> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusPersonDeserializer.class);
-
- /**
- * Because the GooglePlus Person object contains complex objects within its hierarchy, we have to use
- * a custom deserializer
- *
- * @param jsonParser jsonParser
- * @param deserializationContext deserializationContext
- * @return The deserialized {@link com.google.api.services.plus.model.Person} object
- * @throws IOException IOException
- * @throws JsonProcessingException JsonProcessingException
- */
- @Override
- public Person deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- JsonNode node = jsonParser.getCodec().readTree(jsonParser);
- Person person = new Person();
- try {
- person.setId(node.get("id").asText());
- person.setCircledByCount((Integer) (node.get("circledByCount")).numberValue());
- person.setDisplayName(node.get("displayName").asText());
- if( node.has("etag")) {
- person.setEtag(node.get("etag").asText());
- }
- if( node.has("gender")) {
- person.setGender(node.get("gender").asText());
- }
-
- Person.Image image = new Person.Image();
- if( node.has("image") ) {
- JsonNode imageNode = node.get("image");
- image.setIsDefault(imageNode.get("isDefault").asBoolean());
- image.setUrl(imageNode.get("url").asText());
- person.setImage(image);
- }
-
- person.setIsPlusUser(node.get("isPlusUser").asBoolean());
- person.setKind(node.get("kind").asText());
-
- JsonNode nameNode = node.get("name");
- Person.Name name = mapper.readValue(mapper.writeValueAsString(nameNode), Person.Name.class);
- person.setName(name);
-
- person.setObjectType(node.get("objectType").asText());
-
- List<Person.Organizations> organizations = new ArrayList<>();
- if( node.has("organizations")) {
- for (JsonNode orgNode : node.get("organizations")) {
- Person.Organizations org = mapper.readValue(mapper.writeValueAsString(orgNode), Person.Organizations.class);
- organizations.add(org);
- }
- person.setOrganizations(organizations);
- }
-
- person.setUrl(node.get("url").asText());
- person.setVerified(node.get("verified").asBoolean());
-
- List<Person.Emails> emails = new ArrayList<>();
-
- if ( node.has("emails")) {
- for (JsonNode emailNode : node.get("emails")) {
- Person.Emails email = mapper.readValue(mapper.writeValueAsString(emailNode), Person.Emails.class);
- emails.add(email);
- }
- }
-
- if ( node.has("tagline")) {
- person.setTagline(node.get("tagline").asText());
- }
- if ( node.has("aboutMe")) {
- person.setAboutMe(node.get("aboutMe").asText());
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserialize a Person object: {}", ex);
- }
-
- return person;
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GooglePlusActivityUtil.java b/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GooglePlusActivityUtil.java
deleted file mode 100644
index 7039266..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/java/org/apache/streams/gplus/serializer/util/GooglePlusActivityUtil.java
+++ /dev/null
@@ -1,298 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.serializer.util;
-
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.pojo.json.Provider;
-
-import com.google.api.services.plus.model.Comment;
-import com.google.api.services.plus.model.Person;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * GooglePlusActivityUtil helps convert c.g.Person and c.g.Activity into o.a.s.p.j.o.Page and o.a.s.p.j.Activity.
- */
-public class GooglePlusActivityUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusActivityUtil.class);
-
- /**
- * Given a {@link Person} object and an
- * {@link Activity} object, fill out the appropriate details.
- *
- * @param item Person
- * @param activity Activity
- * @throws ActivitySerializerException ActivitySerializerException
- */
- public static void updateActivity(Person item, Activity activity) throws ActivitySerializerException {
- activity.setActor(buildActor(item));
- activity.setVerb("update");
-
- activity.setId(formatId(activity.getVerb(), Optional.ofNullable(item.getId()).orElse(null)));
-
- activity.setProvider(getProvider());
- }
-
- /**
- * Given a {@link List} of {@link Comment} objects and an
- * {@link Activity}, update that Activity to contain all comments
- *
- * @param comments input List of Comment
- * @param activity output Activity
- */
- public static void updateActivity(List<Comment> comments, Activity activity) {
- for (Comment comment : comments) {
- addComment(activity, comment);
- }
-
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- extensions.put("comment_count", comments.size());
- }
-
- /**
- * Given a Google Plus {@link com.google.api.services.plus.model.Activity},
- * convert that into an Activity streams formatted {@link Activity}
- *
- * @param gPlusActivity input c.g.a.s.p.m.Activity
- * @param activity output o.a.s.p.j.Activity
- */
- public static void updateActivity(com.google.api.services.plus.model.Activity gPlusActivity, Activity activity) {
- activity.setActor(buildActor(gPlusActivity.getActor()));
- activity.setVerb("post");
- activity.setTitle(gPlusActivity.getTitle());
- activity.setUrl(gPlusActivity.getUrl());
- activity.setProvider(getProvider());
-
- if (gPlusActivity.getObject() != null) {
- activity.setContent(gPlusActivity.getObject().getContent());
- }
-
- activity.setId(formatId(activity.getVerb(), Optional.ofNullable(gPlusActivity.getId()).orElse(null)));
-
- DateTime published = new DateTime(String.valueOf(gPlusActivity.getPublished()));
- activity.setPublished(published);
-
- setObject(activity, gPlusActivity.getObject());
- addGPlusExtensions(activity, gPlusActivity);
- }
-
- /**
- * Adds a single {@link Comment} to the Object.Attachments
- * section of the passed in {@link Activity}
- *
- * @param activity output o.a.s.p.j.Activity
- * @param comment input c.g.a.s.p.m.Comment
- */
- private static void addComment(Activity activity, Comment comment) {
- ActivityObject obj = new ActivityObject();
-
- obj.setId(comment.getId());
- obj.setPublished(new DateTime(String.valueOf(comment.getPublished())));
- obj.setUpdated(new DateTime(String.valueOf(comment.getUpdated())));
- obj.setContent(comment.getObject().getContent());
- obj.setObjectType(comment.getObject().getObjectType());
-
- Map<String, Object> extensions = new HashMap<>();
- extensions.put("googlePlus", comment);
-
- obj.setAdditionalProperty("extensions", extensions);
-
- if (activity.getObject() == null) {
- activity.setObject(new ActivityObject());
- }
- if (activity.getObject().getAttachments() == null) {
- activity.getObject().setAttachments(new ArrayList<>());
- }
-
- activity.getObject().getAttachments().add(obj);
- }
-
- /**
- * Add in necessary extensions from the passed in {@link com.google.api.services.plus.model.Activity} to the
- * {@link Activity} object
- *
- * @param activity output o.a.s.p.j.Activity
- * @param gPlusActivity input c.g.a.s.p.m.Activity
- */
- private static void addGPlusExtensions(Activity activity, com.google.api.services.plus.model.Activity gPlusActivity) {
-
- activity.getAdditionalProperties().put("googlePlus", gPlusActivity);
-
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
-
- com.google.api.services.plus.model.Activity.PlusObject object = gPlusActivity.getObject();
-
- if (object != null) {
- com.google.api.services.plus.model.Activity.PlusObject.Plusoners plusoners = object.getPlusoners();
- if (plusoners != null) {
- Map<String, Object> likes = new HashMap<>();
- likes.put("count", plusoners.getTotalItems());
- extensions.put("likes", likes);
- }
-
- com.google.api.services.plus.model.Activity.PlusObject.Resharers resharers = object.getResharers();
- if (resharers != null) {
- Map<String, Object> rebroadcasts = new HashMap<>();
- rebroadcasts.put("count", resharers.getTotalItems());
- extensions.put("rebroadcasts", rebroadcasts);
- }
-
- extensions.put("keywords", object.getContent());
- }
- }
-
- /**
- * Set the {@link ActivityObject} field given the passed in
- * {@link com.google.api.services.plus.model.Activity.PlusObject}
- *
- * @param activity output $.object as o.a.s.p.j.ActivityObject
- * @param plusObject input c.g.a.s.p.m.Activity.PlusObject
- */
- private static void setObject(Activity activity, com.google.api.services.plus.model.Activity.PlusObject plusObject) {
- if (plusObject != null) {
- ActivityObject activityObject = new ActivityObject();
-
- activityObject.setContent(plusObject.getContent());
- activityObject.setObjectType(plusObject.getObjectType());
-
- List<ActivityObject> attachmentsList = new ArrayList<>();
- for (com.google.api.services.plus.model.Activity.PlusObject.Attachments attachments : plusObject.getAttachments()) {
- ActivityObject attach = new ActivityObject();
-
- attach.setContent(attachments.getContent());
- attach.setDisplayName(attachments.getDisplayName());
- attach.setObjectType(attachments.getObjectType());
- attach.setUrl(attachments.getUrl());
-
- Image image = new Image();
- com.google.api.services.plus.model.Activity.PlusObject.Attachments.Image image1 = attachments.getImage();
-
- if (image1 != null) {
- image.setUrl(image1.getUrl());
- attach.setImage(image);
- }
-
- attachmentsList.add(attach);
- }
-
- activityObject.setAttachments(attachmentsList);
-
- activity.setObject(activityObject);
- }
- }
-
- /**
- * Given a {@link com.google.api.services.plus.model.Activity.Actor} object, return a fully fleshed
- * out {@link ActivityObject} actor
- *
- * @param gPlusActor input c.g.a.s.p.m.Activity.Actor
- * @return {@link ActivityObject} output $.actor as o.a.s.p.j.ActivityObject
- */
- private static ActivityObject buildActor(com.google.api.services.plus.model.Activity.Actor gPlusActor) {
- ActivityObject actor = new ActivityObject();
-
- actor.setDisplayName(gPlusActor.getDisplayName());
- actor.setId(formatId(String.valueOf(gPlusActor.getId())));
- actor.setUrl(gPlusActor.getUrl());
-
- Image image = new Image();
- com.google.api.services.plus.model.Activity.Actor.Image googlePlusImage = gPlusActor.getImage();
-
- if (googlePlusImage != null) {
- image.setUrl(googlePlusImage.getUrl());
- }
- actor.setImage(image);
-
- return actor;
- }
-
- /**
- * Extract the relevant details from the passed in {@link Person} object and build
- * an actor with them
- *
- * @param person Person
- * @return Actor constructed with relevant Person details
- */
- private static ActivityObject buildActor(Person person) {
- ActivityObject actor = new ActivityObject();
-
- actor.setUrl(person.getUrl());
- actor.setDisplayName(person.getDisplayName());
- actor.setId(formatId(String.valueOf(person.getId())));
-
- if (person.getAboutMe() != null) {
- actor.setSummary(person.getAboutMe());
- } else if (person.getTagline() != null) {
- actor.setSummary(person.getTagline());
- }
-
- Image image = new Image();
- Person.Image googlePlusImage = person.getImage();
-
- if (googlePlusImage != null) {
- image.setUrl(googlePlusImage.getUrl());
- }
- actor.setImage(image);
-
- Map<String, Object> extensions = new HashMap<>();
-
- extensions.put("followers", person.getCircledByCount());
- extensions.put("googleplus", person);
- actor.setAdditionalProperty("extensions", extensions);
-
- return actor;
- }
-
- /**
- * Gets the common googleplus {@link Provider} object
- * @return a provider object representing GooglePlus
- */
- public static Provider getProvider() {
- Provider provider = new Provider();
- provider.setId("id:providers:googleplus");
- provider.setDisplayName("GooglePlus");
- return provider;
- }
-
- /**
- * Formats the ID to conform with the Apache Streams activity ID convention
- * @param idparts the parts of the ID to join
- * @return a valid Activity ID in format "id:googleplus:part1:part2:...partN"
- */
- public static String formatId(String... idparts) {
- return String.join(":",
- Stream.concat(Arrays.stream(new String[]{"id:googleplus"}), Arrays.stream(idparts)).collect(Collectors.toList()));
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/org/apache/streams/google/gplus/GPlusConfiguration.json b/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/org/apache/streams/google/gplus/GPlusConfiguration.json
deleted file mode 100644
index 3b7795b..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/jsonschema/org/apache/streams/google/gplus/GPlusConfiguration.json
+++ /dev/null
@@ -1,101 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.google.gplus.GPlusConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "protocol": {
- "type": "string",
- "description": "The protocol"
- },
- "host": {
- "type": "string",
- "description": "The host"
- },
- "port": {
- "type": "integer",
- "description": "The port"
- },
- "version": {
- "type": "string",
- "description": "The version"
- },
- "endpoint": {
- "type": "string",
- "description": "The endpoint"
- },
- "follow": {
- "type": "array",
- "description": "DEPRECATED. A list of user names, indicating the users whose activities should be delivered on the stream",
- "items": {
- "type": "string"
- }
- },
- "googlePlusUsers": {
- "type": "array",
- "description": "A list of user user ids and optional date parameters for the GPlus provider",
- "items": {
- "type": "object",
- "$ref": "#/definitions/userInfo"
- }
- },
- "defaultAfterDate": {
- "type": "string",
- "format": "date-time",
- "description": "Optional parameter for the provider. If this value is not null an the afterDate value in the userInfo is null, this value will be used."
- },
- "defaultBeforeDate": {
- "type": "string",
- "format": "date-time",
- "description": "Optional parameter for the provider. If this value is not null and the beforeDate value in the userInfo is null, this value will be used."
- },
- "oauth": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.google.gplus.GPlusOAuthConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "DEPRICATED",
- "properties": {
- "appName": {
- "type": "string"
- },
- "pathToP12KeyFile": {
- "type": "string",
- "description": "Absolute Path to key file"
- },
- "serviceAccountEmailAddress": {
- "type": "string",
- "description": "Service Account email address for your app"
- }
- }
- }
- },
- "definitions": {
- "userInfo": {
- "type": "object",
- "javaInterfaces" : ["java.io.Serializable"],
- "dynamic": "true",
- "javaType": "org.apache.streams.google.gplus.configuration.UserInfo",
- "properties": {
- "userId": {
- "type": "string",
- "description": "Google+ user id"
- },
- "afterDate": {
- "type": "string",
- "format": "date-time",
- "description": "If the api allows to gather data by date range, this date will be used as the start of the range for the request for this user. If this is null it will use the defaultBeforeDate."
- },
- "beforeDate": {
- "type": "string",
- "format": "date-time",
- "description": "If the api allows to gather data by date range, this date will be used as the end of the range for the request for this user.. If this is null it will use the defaultAfterDate."
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/resources/components.dot b/streams-contrib/streams-provider-google/google-gplus/src/main/resources/components.dot
deleted file mode 100644
index ed631f2..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/resources/components.dot
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- subgraph cluster_generators {
- label="generators";
- style="dotted";
- generators_gplus [label="https://www.googleapis.com/gmail/v1"]
- }
-
- subgraph cluster_providers {
- label="providers";
- providers_gplus_GPlusUserActivityProvider [label="GPlusUserActivityProvider"]
- providers_gplus_GPlusUserDataProvider [label="providers_gplus_GPlusUserDataProvider"]
- }
-
- subgraph cluster_processors {
- label="processors";
- processors_gplus_activity [label="GooglePlusTypeConverter"]
- }
-
- persisters [label="persisters", shape="cylinder"];
-
- generators_gplus -> providers_gplus_GPlusUserActivityProvider
- generators_gplus -> providers_gplus_GPlusUserDataProvider
-
- providers_gplus_GPlusUserActivityProvider -> processors_gplus_activity [label="c.g.a.s.p.m.Activity"]
- providers_gplus_GPlusUserDataProvider -> processors_gplus_activity [label="c.g.a.s.p.m.Person"]
-
- processors_gplus_activity -> persisters [label="o.a.s.p.j.Activity"]
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/main/resources/reference.conf b/streams-contrib/streams-provider-google/google-gplus/src/main/resources/reference.conf
deleted file mode 100644
index fbf456f..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/main/resources/reference.conf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-gplus {
- protocol = "https"
- host = "www.googleapis.com/plus"
- port = 443
- version = "v1"
- endpoint = "people/me/activities/public?maxResults=100"
- filter-level = "none"
- oauth {
- appName = "Apache Streams"
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/site/markdown/index.md b/streams-contrib/streams-provider-google/google-gplus/src/site/markdown/index.md
deleted file mode 100644
index ec5bcee..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/site/markdown/index.md
+++ /dev/null
@@ -1,47 +0,0 @@
-org.apache.streams:google-gplus
-===============================
-
-google-gplus contains providers, conversions, and utility classes for activity exchange with Google+
-
-## Configuration
-
-| Schema | JavaDoc | Example Configuration(s) |
-|--------|---------|--------------------------|
-| [GPlusConfiguration.json](com/google/gplus/GPlusConfiguration.json "GPlusConfiguration.json") | [GPlusConfiguration.html](apidocs/com/google/gplus/GPlusConfiguration.html "GPlusConfiguration.html") | [reference.conf](reference.conf "reference.conf") |
-
-## Components
-
-
-
-| Class | Class JavaDoc |
-|-------|---------------|
-| GPlusUserDataProvider | [GPlusUserDataProvider.html](apidocs/com/google/gplus/provider/GPlusUserDataProvider.html "GPlusUserDataProvider.html") |
-| GPlusUserActivityProvider | [GPlusUserActivityProvider.html](apidocs/com/google/gplus/provider/GPlusUserActivityProvider.html "GPlusUserActivityProvider.html") |
-| GooglePlusTypeConverter | [GooglePlusTypeConverter.html](apidocs/com/google/gplus/processor/GooglePlusTypeConverter.html "GooglePlusTypeConverter.html") |
-
-Test:
------
-
-Log into admin console
-Create project
-Enable Data API on project
-Create service account
-Download p12 file
-
-Create a local file `gplus.conf` with valid gplus credentials
-
- gplus {
- apiKey = ""
- oauth {
- serviceAccountEmailAddress = ""
- pathToP12KeyFile = ""
- }
- }
-
-Build with integration testing enabled, using your credentials
-
- mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/gplus.conf"
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusCommentSerDeIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusCommentSerDeIT.java
deleted file mode 100644
index 1fb79ca..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusCommentSerDeIT.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus;
-
-import org.apache.streams.gplus.serializer.util.GPlusCommentDeserializer;
-import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.plus.model.Comment;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNull;
-
-/**
- * Tests conversion of gplus inputs to Activity.
- */
-public class GooglePlusCommentSerDeIT {
- private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusCommentSerDeIT.class);
- private ObjectMapper objectMapper;
-
- /**
- * setup.
- */
- @BeforeClass
- public void setupTestCommentObjects() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Comment.class, new GPlusCommentDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- @Test
- public void testCommentObjects() {
- InputStream is = GooglePlusCommentSerDeIT.class.getResourceAsStream("/google_plus_comments_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- Activity activity = new Activity();
- List<Comment> comments = new ArrayList<>();
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Comment comment = objectMapper.readValue(line, Comment.class);
-
- LOGGER.info("comment: {}", comment);
-
- assertNotNull(comment);
- assertNotNull(comment.getEtag());
- assertNotNull(comment.getId());
- assertNotNull(comment.getInReplyTo());
- assertNotNull(comment.getObject());
- assertNotNull(comment.getPlusoners());
- assertNotNull(comment.getPublished());
- assertNotNull(comment.getUpdated());
- assertNotNull(comment.getSelfLink());
- assertEquals(comment.getVerb(), "post");
-
- comments.add(comment);
- }
- }
-
- assertEquals(comments.size(), 3);
-
- GooglePlusActivityUtil.updateActivity(comments, activity);
- assertNotNull(activity);
- assertNotNull(activity.getObject());
- assertEquals(activity.getObject().getAttachments().size(), 3);
- } catch (Exception ex) {
- LOGGER.error("Exception while testing serializability: {}", ex);
- }
- }
-
- @Test
- public void testEmptyComments() {
- Activity activity = new Activity();
- GooglePlusActivityUtil.updateActivity(new ArrayList<>(), activity);
- assertNull(activity.getObject());
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusPersonSerDeIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusPersonSerDeIT.java
deleted file mode 100644
index ce53bf3..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/GooglePlusPersonSerDeIT.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus;
-
-import org.apache.streams.gplus.serializer.util.GPlusPersonDeserializer;
-import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Provider;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.plus.model.Person;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-
-/**
- * Tests conversion of gplus inputs to Activity.
- */
-public class GooglePlusPersonSerDeIT {
- private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusPersonSerDeIT.class);
- private ObjectMapper objectMapper;
-
- /**
- * setup.
- */
- @BeforeClass
- public void setup() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- @Test
- public void testPersonObjects() {
- InputStream is = GooglePlusPersonSerDeIT.class.getResourceAsStream("/google_plus_person_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Activity activity = new Activity();
-
- Person person = objectMapper.readValue(line, Person.class);
-
- GooglePlusActivityUtil.updateActivity(person, activity);
- LOGGER.info("activity: {}", activity);
-
- assertNotNull(activity);
- assertTrue(activity.getId().contains("id:googleplus:update"));
- assertEquals(activity.getVerb(), "update");
-
- Provider provider = activity.getProvider();
- assertEquals(provider.getId(), "id:providers:googleplus");
- assertEquals(provider.getDisplayName(), "GooglePlus");
-
- ActivityObject actor = activity.getActor();
- assertNotNull(actor.getImage());
- assertTrue(actor.getId().contains("id:googleplus:"));
- assertNotNull(actor.getUrl());
-
- }
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while testing serializability: {}", ex);
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusActivitySerDeIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusActivitySerDeIT.java
deleted file mode 100644
index b7bb3bf..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusActivitySerDeIT.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.processors;
-
-import org.apache.streams.gplus.serializer.util.GPlusActivityDeserializer;
-import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Provider;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.Map;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-
-/**
- * Tests conversion of gplus inputs to Activity.
- */
-public class GooglePlusActivitySerDeIT {
- private static final Logger LOGGER = LoggerFactory.getLogger(GooglePlusActivitySerDeIT.class);
- private ObjectMapper objectMapper;
-
- /**
- * setup.
- */
- @BeforeClass
- public void setup() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(com.google.api.services.plus.model.Activity.class, new GPlusActivityDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testActivityObjects() {
- InputStream is = GooglePlusActivitySerDeIT.class.getResourceAsStream("/google_plus_activity_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Activity activity = new Activity();
-
- com.google.api.services.plus.model.Activity googlePlusActivity =
- objectMapper.readValue(line, com.google.api.services.plus.model.Activity.class);
-
- GooglePlusActivityUtil.updateActivity(googlePlusActivity, activity);
- LOGGER.info("activity: {}", activity);
-
- assertNotNull(activity);
- assert (activity.getId().contains("id:googleplus:post"));
- assertEquals(activity.getVerb(), "post");
-
- Provider provider = activity.getProvider();
- assertEquals(provider.getId(), "id:providers:googleplus");
- assertEquals(provider.getDisplayName(), "GooglePlus");
-
- ActivityObject actor = activity.getActor();
- assertNotNull(actor.getImage());
- assert (actor.getId().contains("id:googleplus:"));
- assertNotNull(actor.getUrl());
-
- assertNotNull(activity.getPublished());
- assertNotNull(activity.getTitle());
- assertNotNull(activity.getUrl());
-
- Map<String, Object> extensions = ExtensionUtil.getInstance().getExtensions(activity);
- assertNotNull(extensions);
-
- if (activity.getContent() != null) {
- assertNotNull(extensions.get("rebroadcasts"));
- assertNotNull(extensions.get("keywords"));
- assertNotNull(extensions.get("likes"));
- assert (((Map<String, Object>) extensions.get("rebroadcasts")).containsKey("count"));
- assert (((Map<String, Object>) extensions.get("likes")).containsKey("count"));
- }
- }
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while testing serializability: {}", ex);
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusTypeConverterIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusTypeConverterIT.java
deleted file mode 100644
index d3472c4..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/processors/GooglePlusTypeConverterIT.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.processors;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.gplus.processor.GooglePlusTypeConverter;
-import org.apache.streams.gplus.serializer.util.GPlusActivityDeserializer;
-import org.apache.streams.gplus.serializer.util.GPlusPersonDeserializer;
-import org.apache.streams.gplus.serializer.util.GooglePlusActivityUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.plus.model.Person;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.List;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-
-/**
- * Tests conversion of gplus inputs to Activity
- */
-public class GooglePlusTypeConverterIT {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(GooglePlusTypeConverterIT.class);
- private GooglePlusTypeConverter googlePlusTypeConverter;
- private ObjectMapper objectMapper;
-
- @BeforeClass
- public void setup() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Person.class, new GPlusPersonDeserializer());
- simpleModule.addDeserializer(com.google.api.services.plus.model.Activity.class, new GPlusActivityDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
- googlePlusTypeConverter = new GooglePlusTypeConverter();
- googlePlusTypeConverter.prepare(null);
- }
-
- @Test(dependsOnGroups = {"testGPlusUserDataProvider"})
- public void testProcessPerson() throws IOException, ActivitySerializerException {
-
- File file = new File("target/test-classes/GPlusUserDataProviderIT.stdout.txt");
- InputStream is = new FileInputStream(file);
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Activity activity = new Activity();
-
- Person person = objectMapper.readValue(line, Person.class);
- StreamsDatum streamsDatum = new StreamsDatum(person);
-
- assertNotNull(streamsDatum.getDocument());
-
- List<StreamsDatum> retList = googlePlusTypeConverter.process(streamsDatum);
- GooglePlusActivityUtil.updateActivity(person, activity);
-
- assertEquals(retList.size(), 1);
- assert (retList.get(0).getDocument() instanceof Activity);
- assertEquals(activity, retList.get(0).getDocument());
- }
- }
- }
-
- @Test(dependsOnGroups = {"testGPlusUserActivityProvider"})
- public void testProcessActivity() throws IOException, ActivitySerializerException {
-
- File file = new File("target/test-classes/GPlusUserActivityProviderIT.stdout.txt");
- InputStream is = new FileInputStream(file);
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- Activity activity = new Activity();
-
- com.google.api.services.plus.model.Activity gPlusActivity = objectMapper.readValue(line, com.google.api.services.plus.model.Activity.class);
- StreamsDatum streamsDatum = new StreamsDatum(gPlusActivity);
-
- assertNotNull(streamsDatum.getDocument());
-
- List<StreamsDatum> retList = googlePlusTypeConverter.process(streamsDatum);
- GooglePlusActivityUtil.updateActivity(gPlusActivity, activity);
-
- assertEquals(retList.size(), 1);
- assertTrue(retList.get(0).getDocument() instanceof Activity);
- assertEquals(activity, retList.get(0).getDocument());
- }
- }
- }
-
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserActivityProviderIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserActivityProviderIT.java
deleted file mode 100644
index 9f8524d..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserActivityProviderIT.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.providers;
-
-import org.apache.streams.gplus.provider.GPlusUserActivityProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-public class GPlusUserActivityProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserActivityProviderIT.class);
-
- @Test(groups={"testGPlusUserActivityProvider"})
- public void testGPlusUserActivityProvider() throws Exception {
-
- String configfile = "./target/test-classes/GPlusUserActivityProviderIT.conf";
- String outfile = "./target/test-classes/GPlusUserActivityProviderIT.stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- GPlusUserActivityProvider.main(args);
- } catch ( Exception ex ) {
- LOGGER.error("Test Exception!", ex);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- File out = new File(outfile);
- assert (out.exists());
- assert (out.canRead());
- assert (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- assert (outCounter.getLineNumber() >= 1);
-
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserDataProviderIT.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserDataProviderIT.java
deleted file mode 100644
index 012cdc2..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/GPlusUserDataProviderIT.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.providers;
-
-import org.apache.streams.gplus.provider.GPlusUserDataProvider;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-public class GPlusUserDataProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(GPlusUserDataProviderIT.class);
-
- @Test(groups={"testGPlusUserDataProvider"})
- public void testGPlusUserDataProvider() throws Exception {
-
- String configfile = "./target/test-classes/GPlusUserDataProviderIT.conf";
- String outfile = "./target/test-classes/GPlusUserDataProviderIT.stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- GPlusUserDataProvider.main(args);
- } catch ( Exception ex ) {
- LOGGER.error("Test Exception!", ex);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- GPlusUserDataProvider.main(new String[]{configfile, outfile});
-
- File out = new File(outfile);
- assert (out.exists());
- assert (out.canRead());
- assert (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- assert (outCounter.getLineNumber() >= 1);
-
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestAbstractGPlusProvider.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestAbstractGPlusProvider.java
deleted file mode 100644
index 61364af..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestAbstractGPlusProvider.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.providers;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.GPlusConfiguration;
-import org.apache.streams.google.gplus.GPlusOAuthConfiguration;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.gplus.provider.AbstractGPlusProvider;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import com.carrotsearch.randomizedtesting.annotations.Repeat;
-import com.google.api.services.plus.Plus;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-import static org.mockito.Mockito.mock;
-
-import org.junit.Assert;
-
-
-/**
- * Unit tests for {@link org.apache.streams.gplus.provider.AbstractGPlusProvider}
- */
-public class TestAbstractGPlusProvider extends RandomizedTest {
-
- /**
- * Test that every collector will be run and that data queued from the collectors will be processed.
- */
- @Test
- @Repeat(iterations = 3)
- public void testDataCollectorRunsPerUser() {
- int numUsers = randomIntBetween(1, 1000);
- List<UserInfo> userList = new LinkedList<>();
- for (int i = 0; i < numUsers; ++i) {
- userList.add(new UserInfo());
- }
- GPlusConfiguration config = new GPlusConfiguration();
- GPlusOAuthConfiguration oauth = new GPlusOAuthConfiguration();
- oauth.setAppName("a");
- oauth.setPathToP12KeyFile("a");
- oauth.setServiceAccountEmailAddress("a");
- config.setOauth(oauth);
- config.setGooglePlusUsers(userList);
- AbstractGPlusProvider provider = new AbstractGPlusProvider(config) {
-
- @Override
- protected Plus createPlusClient() throws IOException {
- return mock(Plus.class);
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, Plus plus, UserInfo userInfo) {
- final BlockingQueue<StreamsDatum> q = queue;
- return () -> {
- try {
- q.put(new StreamsDatum(null));
- } catch (InterruptedException ie) {
- Assert.fail("Test was interrupted");
- }
- };
- }
- };
-
- try {
- provider.prepare(null);
- provider.startStream();
- int datumCount = 0;
- while (provider.isRunning()) {
- datumCount += provider.readCurrent().size();
- }
- Assert.assertEquals(numUsers, datumCount);
- } finally {
- provider.cleanUp();
- }
- }
-
-
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserActivityCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserActivityCollector.java
deleted file mode 100644
index 47ac3f6..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserActivityCollector.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.providers;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.gplus.provider.GPlusUserActivityCollector;
-import org.apache.streams.gplus.serializer.util.GPlusActivityDeserializer;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.impl.ConstantTimeBackOffStrategy;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import com.carrotsearch.randomizedtesting.annotations.Repeat;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.plus.Plus;
-import com.google.api.services.plus.model.Activity;
-import com.google.api.services.plus.model.ActivityFeed;
-import org.joda.time.DateTime;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.junit.Assert;
-
-
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Unit tests for {@link org.apache.streams.gplus.provider.GPlusUserActivityCollector}
- */
-public class TestGPlusUserActivityCollector extends RandomizedTest {
-
- private static final String ACTIVITY_TEMPLATE = "{ \"kind\": \"plus#activity\", \"etag\": \"\\\"Vea_b94Y77GDGgRK7gFNPnolKQw/v1-6aVSBGT4qiStMoz7f2_AN2fM\\\"\", \"title\": \"\", \"published\": \"%s\", \"updated\": \"2014-10-27T06:26:33.927Z\", \"id\": \"z13twrlznpvtzz52w22mdt1y0k3of1djw04\", \"url\": \"https://plus.google.com/116771159471120611293/posts/GR7CGR8N5VL\", \"actor\": { \"id\": \"116771159471120611293\", \"displayName\": \"displayName\", \"url\": \"https://plus.google.com/1167 [...]
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
- private static final String IN_RANGE_IDENTIFIER = "data in range";
-
- static {
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Activity.class, new GPlusActivityDeserializer());
- MAPPER.registerModule(simpleModule);
- MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- private static ActivityFeed createMockActivityFeed(
- int numBefore,
- int numAfter,
- int numInRange,
- DateTime after,
- DateTime before,
- boolean page) {
- ActivityFeed feed = new ActivityFeed();
- List<Activity> list = new LinkedList<>();
- for (int i = 0; i < numAfter; ++i) {
- DateTime published = before.plus(randomIntBetween(0, Integer.MAX_VALUE));
- Activity activity = createActivityWithPublishedDate(published);
- list.add(activity);
- }
- for (int i = 0; i < numInRange; ++i) {
- DateTime published;
- if ((before == null && after == null) || before == null) {
- published = DateTime.now(); // no date range or end time date range so just make the time now.
- } else if (after == null) {
- published = before.minusMillis(randomIntBetween(1, Integer.MAX_VALUE)); //no beginning to range
- } else { // has to be in range
- long range = before.getMillis() - after.getMillis();
- published = after.plus(range / 2); //in the middle
- }
- Activity activity = createActivityWithPublishedDate(published);
- activity.setTitle(IN_RANGE_IDENTIFIER);
- list.add(activity);
- }
- for (int i = 0; i < numBefore; ++i) {
- DateTime published = after.minusMillis(randomIntBetween(1, Integer.MAX_VALUE));
- Activity activity = createActivityWithPublishedDate(published);
- list.add(activity);
- }
- if (page) {
- feed.setNextPageToken("A");
- } else {
- feed.setNextPageToken(null);
- }
- feed.setItems(list);
- return feed;
- }
-
- private static Activity createActivityWithPublishedDate(DateTime dateTime) {
- Activity activity = new Activity();
- activity.setPublished(new com.google.api.client.util.DateTime(dateTime.getMillis()));
- activity.setId("a");
- return activity;
- }
-
- /**
- * Creates a randomized activity and randomized date range.
- * <p/>
- * The activity feed is separated into three chunks,
- * |. . . data too recent to be in date range . . .||. . . data in date range. . .||. . . data too old to be in date range|
- * [index 0, ............................................................................................., index length-1]
- * <p/>
- * Inside of those chunks data has no order, but the list is ordered by those three chunks.
- * <p/>
- * The test will check to see if the num of data in the date range make onto the output queue.
- */
- @Test
- @Repeat(iterations = 3)
- public void testWithBeforeAndAfterDates() throws InterruptedException {
- //initialize counts assuming no date ranges will be used
- int numActivities = randomIntBetween(0, 1000);
- int numActivitiesInDateRange = numActivities;
- int numberOutOfRange = 0;
- int numBeforeRange = 0;
- int numAfterRange = 0;
- //determine if date ranges will be used
- DateTime beforeDate = null;
- DateTime afterDate = null;
- if (randomInt() % 2 == 0) {
- beforeDate = DateTime.now().minusDays(randomIntBetween(1, 5));
- }
- if (randomInt() % 2 == 0) {
- if (beforeDate == null) {
- afterDate = DateTime.now().minusDays(randomIntBetween(1, 10));
- } else {
- afterDate = beforeDate.minusDays(randomIntBetween(1, 10));
- }
- }
- //update counts if date ranges are going to be used.
- if (beforeDate != null || afterDate != null) { //assign amount to be in range
- numActivitiesInDateRange = randomIntBetween(0, numActivities);
- numberOutOfRange = numActivities - numActivitiesInDateRange;
- }
- if (beforeDate == null && afterDate != null) { //assign all out of range to be before the start of the range
- numBeforeRange = numberOutOfRange;
- } else if (beforeDate != null && afterDate == null) { //assign all out of range to be after the start of the range
- numAfterRange = numberOutOfRange;
- } else if (beforeDate != null && afterDate != null) { //assign half before range and half after the range
- numAfterRange = (numberOutOfRange / 2) + (numberOutOfRange % 2);
- numBeforeRange = numberOutOfRange / 2;
- }
-
- Plus plus = createMockPlus(numBeforeRange, numAfterRange, numActivitiesInDateRange, afterDate, beforeDate);
- BackOffStrategy strategy = new ConstantTimeBackOffStrategy(1);
- BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
- UserInfo userInfo = new UserInfo();
- userInfo.setUserId("A");
- userInfo.setAfterDate(afterDate);
- userInfo.setBeforeDate(beforeDate);
- GPlusUserActivityCollector collector = new GPlusUserActivityCollector(plus, datums, strategy, userInfo);
- collector.run();
-
- Assert.assertEquals(numActivitiesInDateRange, datums.size());
- while (!datums.isEmpty()) {
- StreamsDatum datum = datums.take();
- Assert.assertNotNull(datum);
- Assert.assertNotNull(datum.getDocument());
- Assert.assertTrue(datum.getDocument() instanceof String);
- Assert.assertTrue(((String)datum.getDocument()).contains(IN_RANGE_IDENTIFIER)); //only in range documents are on the out going queue.
- }
- }
-
- private Plus createMockPlus(final int numBefore, final int numAfter, final int numInRange, final DateTime after, final DateTime before) {
- Plus plus = mock(Plus.class);
- final Plus.Activities activities = createMockPlusActivities(numBefore, numAfter, numInRange, after, before);
- doAnswer(invocationOnMock -> activities).when(plus).activities();
- return plus;
- }
-
- private Plus.Activities createMockPlusActivities(
- final int numBefore,
- final int numAfter,
- final int numInRange,
- final DateTime after,
- final DateTime before) {
- Plus.Activities activities = mock(Plus.Activities.class);
- try {
- Plus.Activities.List list = createMockPlusActivitiesList(numBefore, numAfter, numInRange, after, before);
- when(activities.list(anyString(), anyString())).thenReturn(list);
- } catch (IOException ioe) {
- Assert.fail("Should not have thrown exception while creating mock. : " + ioe.getMessage());
- }
- return activities;
- }
-
- private Plus.Activities.List createMockPlusActivitiesList(
- final int numBefore,
- final int numAfter,
- final int numInRange,
- final DateTime after,
- final DateTime before) {
- Plus.Activities.List list = mock(Plus.Activities.List.class);
- when(list.setMaxResults(anyLong())).thenReturn(list);
- when(list.setPageToken(anyString())).thenReturn(list);
- ActivityFeedAnswer answer = new ActivityFeedAnswer(numBefore, numAfter, numInRange, after, before);
- try {
- doAnswer(answer).when(list).execute();
- } catch (IOException ioe) {
- Assert.fail("Should not have thrown exception while creating mock. : " + ioe.getMessage());
- }
- return list;
- }
-
- private static class ActivityFeedAnswer implements Answer<ActivityFeed> {
- private int afterCount = 0;
- private int beforeCount = 0;
- private int inCount = 0;
- private int maxBatch = 100;
-
- private int numAfter;
- private int numInRange;
- private int numBefore;
- private DateTime after;
- private DateTime before;
-
- private ActivityFeedAnswer(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
- this.numBefore = numBefore;
- this.numAfter = numAfter;
- this.numInRange = numInRange;
- this.after = after;
- this.before = before;
- }
-
- @Override
- public ActivityFeed answer(InvocationOnMock invocationOnMock) throws Throwable {
- int totalCount = 0;
- int batchAfter = 0;
- int batchBefore = 0;
- int batchIn = 0;
- if (afterCount != numAfter) {
- if (numAfter - afterCount >= maxBatch) {
- afterCount += maxBatch;
- batchAfter += maxBatch;
- totalCount += batchAfter;
- } else {
- batchAfter += numAfter - afterCount;
- totalCount += numAfter - afterCount;
- afterCount = numAfter;
- }
- }
- if (totalCount < maxBatch && inCount != numInRange) {
- if (numInRange - inCount >= maxBatch - totalCount) {
- inCount += maxBatch - totalCount;
- batchIn += maxBatch - totalCount;
- totalCount += batchIn;
- } else {
- batchIn += numInRange - inCount;
- totalCount += numInRange - inCount;
- inCount = numInRange;
- }
- }
- if (totalCount < maxBatch && beforeCount != numBefore) {
- if (numBefore - batchBefore >= maxBatch - totalCount) {
- batchBefore += maxBatch - totalCount;
- totalCount = maxBatch;
- beforeCount += batchBefore;
- } else {
- batchBefore += numBefore - beforeCount;
- totalCount += numBefore - beforeCount;
- beforeCount = numBefore;
- }
- }
-
- return createMockActivityFeed(
- batchBefore,
- batchAfter,
- batchIn,
- after,
- before,
- numAfter != afterCount || inCount != numInRange || beforeCount != numBefore);
- }
- }
-
-
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserDataCollector.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserDataCollector.java
deleted file mode 100644
index f81ecc0..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/providers/TestGPlusUserDataCollector.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.providers;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.gplus.provider.GPlusUserDataCollector;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.impl.ConstantTimeBackOffStrategy;
-
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.services.plus.Plus;
-import com.google.api.services.plus.model.Person;
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.io.IOException;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Basic Units for {@link org.apache.streams.gplus.provider.GPlusUserDataCollector}.
- */
-public class TestGPlusUserDataCollector {
-
- private static final String NO_ERROR = "no error";
-
- /**
- * Test that on success a datum will be added to the queue.
- *
- * @throws Exception Exception
- */
- @Test
- public void testSucessfullPull() throws Exception {
- Plus plus = createMockPlus(0, null);
- BackOffStrategy backOff = new ConstantTimeBackOffStrategy(1);
- BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
- UserInfo user = new UserInfo();
- user.setUserId("A");
-
- GPlusUserDataCollector collector = new GPlusUserDataCollector(plus, backOff, datums, user);
- collector.run();
-
- Assert.assertEquals(1, datums.size());
- StreamsDatum datum = datums.take();
- Assert.assertNotNull(datum);
- Assert.assertEquals(NO_ERROR, datum.getId());
- Assert.assertNotNull(datum.getDocument());
- Assert.assertTrue(datum.getDocument() instanceof String);
- }
-
- /**
- * Test that on failure, no datums are output.
- *
- * @throws Exception Exception
- */
- @Test
- public void testFail() throws Exception {
- Plus plus = createMockPlus(3, mock(GoogleJsonResponseException.class));
- UserInfo user = new UserInfo();
- user.setUserId("A");
- BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
- BackOffStrategy backOffStrategy = new ConstantTimeBackOffStrategy(1);
-
- GPlusUserDataCollector collector = new GPlusUserDataCollector(plus, backOffStrategy, datums, user);
- collector.run();
-
- Assert.assertEquals(0, datums.size());
- }
-
- private Plus createMockPlus(final int succedOnTry, final Throwable throwable) {
- Plus plus = mock(Plus.class);
- doAnswer(invocationOnMock -> createMockPeople(succedOnTry, throwable)).when(plus).people();
- return plus;
- }
-
- private Plus.People createMockPeople(final int succedOnTry, final Throwable throwable) {
- Plus.People people = mock(Plus.People.class);
- try {
- when(people.get(anyString())).thenAnswer(invocationOnMock -> createMockGetNoError(succedOnTry, throwable));
- } catch (IOException ioe) {
- Assert.fail("No Excpetion should have been thrown while creating mocks");
- }
- return people;
- }
-
- private Plus.People.Get createMockGetNoError(final int succedOnTry, final Throwable throwable) {
- Plus.People.Get get = mock(Plus.People.Get.class);
- try {
- doAnswer(new Answer() {
- private int counter = 0;
-
- @Override
- public Person answer(InvocationOnMock invocationOnMock) throws Throwable {
- if (counter == succedOnTry) {
- Person person = new Person();
- person.setId(NO_ERROR);
- return person;
- } else {
- ++counter;
- throw throwable;
- }
- }
- }).when(get).execute();
- } catch (IOException ioe) {
- Assert.fail("No Excpetion should have been thrown while creating mocks");
- }
- return get;
- }
-
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifierTest.java b/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifierTest.java
deleted file mode 100644
index 1a9720d..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/java/org/apache/streams/gplus/serializer/util/GPlusEventClassifierTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.gplus.serializer.util;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.api.services.plus.model.Activity;
-import com.google.api.services.plus.model.Person;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * GPlusEventClassifierTest tests GPlusEventClassifier.
- */
-public class GPlusEventClassifierTest {
-
- private static StreamsJacksonMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Test
- public void classifyActivityTest() {
- try {
- Activity activity = new Activity();
- activity.setKind("plus#activity");
- Class retClass = GPlusEventClassifier.detectClass(mapper.writeValueAsString(activity));
- Assert.assertEquals(retClass, Activity.class);
- } catch (Exception ex) {
- //
- }
- }
-
- @Test
- public void classifyPersonTest() {
- try {
- Person person = new Person();
- person.setKind("plus#person");
- Class retClass = GPlusEventClassifier.detectClass(mapper.writeValueAsString(person));
- Assert.assertEquals(retClass, Person.class);
- } catch (Exception ex) {
- //
- }
- }
-
- @Test
- public void classifyObjectNodeTest() {
- try {
- Person person = new Person();
- person.setKind("fake");
- Class retClass = GPlusEventClassifier.detectClass(mapper.writeValueAsString(person));
- Assert.assertEquals(retClass, ObjectNode.class);
- } catch (Exception ex) {
- //
- }
- }
-}
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/GPlusUserActivityProviderIT.conf b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/GPlusUserActivityProviderIT.conf
deleted file mode 100644
index 629ecad..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/GPlusUserActivityProviderIT.conf
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-gplus {
- googlePlusUsers = [
- # organization
- {
- userId = "+apache"
- },
- # person
- {
- userId = "+SteveBlackmon"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/GPlusUserDataProviderIT.conf b/streams-contrib/streams-provider-google/google-gplus/src/test/resources/GPlusUserDataProviderIT.conf
deleted file mode 100644
index 629ecad..0000000
--- a/streams-contrib/streams-provider-google/google-gplus/src/test/resources/GPlusUserDataProviderIT.conf
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-gplus {
- googlePlusUsers = [
- # organization
- {
- userId = "+apache"
- },
- # person
- {
- userId = "+SteveBlackmon"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-google/pom.xml b/streams-contrib/streams-provider-google/pom.xml
deleted file mode 100644
index 50c054d..0000000
--- a/streams-contrib/streams-provider-google/pom.xml
+++ /dev/null
@@ -1,68 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-provider-google</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Google Providers</description>
-
- <packaging>pom</packaging>
-
- <properties>
-
- </properties>
-
- <modules>
- <module>google-gmail</module>
- <module>google-gplus</module>
- </modules>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
-</project>
diff --git a/streams-contrib/streams-provider-instagram/README.md b/streams-contrib/streams-provider-instagram/README.md
deleted file mode 100644
index 372ecc1..0000000
--- a/streams-contrib/streams-provider-instagram/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-provider-instagram
-=============================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-provider-instagram/metadata/InstagramMediaDataActivitySerializer.mup b/streams-contrib/streams-provider-instagram/metadata/InstagramMediaDataActivitySerializer.mup
deleted file mode 100644
index 39c49c4..0000000
--- a/streams-contrib/streams-provider-instagram/metadata/InstagramMediaDataActivitySerializer.mup
+++ /dev/null
@@ -1,897 +0,0 @@
-{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "title": "InstagramMediaDataActivitySerializer",
- "id": 1,
- "formatVersion": 2,
- "ideas": {
- "1": {
- "title": "activity",
- "id": 2,
- "ideas": {
- "1": {
- "title": "published",
- "id": 25
- },
- "2": {
- "title": "id:instagram:post:<id>",
- "id": 26
- },
- "3": {
- "title": "provider",
- "id": 27,
- "ideas": {
- "1": {
- "title": "id",
- "id": 28,
- "ideas": {
- "1": {
- "title": "id:providers:instagram",
- "id": 31
- }
- }
- },
- "2": {
- "title": "displayName",
- "id": 29,
- "ideas": {
- "1": {
- "title": "Instagram",
- "id": 30
- }
- }
- }
- }
- },
- "6": {
- "title": "url",
- "id": 40
- },
- "7": {
- "title": "object",
- "id": 46,
- "ideas": {
- "1": {
- "title": "objectType",
- "id": 38,
- "attr": {
- "style": {}
- }
- },
- "2": {
- "title": "attachments",
- "id": 51,
- "ideas": {
- "1": {
- "title": "object",
- "id": 52,
- "ideas": {
- "1": {
- "title": "objectType",
- "id": 53,
- "ideas": {
- "1": {
- "title": "\"image\"",
- "id": 81
- }
- }
- },
- "2": {
- "title": "image",
- "id": 54,
- "ideas": {
- "1": {
- "title": "url",
- "id": 55
- },
- "2": {
- "title": "height",
- "id": 56
- },
- "3": {
- "title": "width",
- "id": 57
- }
- }
- }
- }
- },
- "2": {
- "title": "object",
- "ideas": {
- "1": {
- "title": "objectType",
- "ideas": {
- "1": {
- "title": "\"image\"",
- "id": 101
- }
- },
- "id": 100
- },
- "2": {
- "title": "image",
- "ideas": {
- "1": {
- "title": "url",
- "id": 103
- },
- "2": {
- "title": "height",
- "id": 104
- },
- "3": {
- "title": "width",
- "id": 105
- }
- },
- "id": 102
- }
- },
- "id": 99,
- "attr": {
- "style": {}
- }
- },
- "3": {
- "title": "object",
- "ideas": {
- "1": {
- "title": "objectType",
- "ideas": {
- "1": {
- "title": "\"video\"",
- "id": 108
- }
- },
- "id": 107
- },
- "2": {
- "title": "image",
- "ideas": {
- "1": {
- "title": "url",
- "id": 110
- },
- "2": {
- "title": "height",
- "id": 111
- },
- "3": {
- "title": "width",
- "id": 112
- }
- },
- "id": 109
- }
- },
- "id": 106,
- "attr": {
- "style": {}
- }
- }
- }
- },
- "3": {
- "title": "image\n\nIF objectType == video\nimage = data.videos.standard_resolution\nELSE \nimage = data.images.standard_resolution",
- "id": 62,
- "attr": {
- "position": [
- 74.5,
- 74,
- 1
- ],
- "style": {}
- },
- "ideas": {
- "1": {
- "title": "url",
- "id": 64
- },
- "2": {
- "title": "height",
- "id": 65
- },
- "3": {
- "title": "width",
- "id": 66
- }
- }
- },
- "4": {
- "title": "content",
- "id": 113
- }
- }
- },
- "8": {
- "title": "extensions",
- "id": 75,
- "ideas": {
- "1": {
- "title": "instagram",
- "id": 91,
- "attr": {
- "style": {}
- }
- },
- "2": {
- "title": "location",
- "id": 77,
- "ideas": {
- "1": {
- "title": "coordinates",
- "id": 78,
- "ideas": {
- "1": {
- "title": "type",
- "id": 79,
- "ideas": {
- "1": {
- "title": "\"Point\"",
- "id": 80
- }
- }
- },
- "0.5": {
- "title": "coordinates\n\n[longitude,latitude]",
- "id": 82,
- "attr": {
- "style": {}
- }
- }
- }
- }
- }
- },
- "12": {
- "title": "hashtags",
- "id": 76,
- "attr": {
- "position": [
- 93.5,
- 125.5,
- 1
- ],
- "style": {}
- }
- },
- "13": {
- "title": "user_mentions",
- "id": 92,
- "ideas": {
- "1": {
- "title": "id",
- "id": 93
- },
- "2": {
- "title": "handle",
- "id": 94
- }
- }
- },
- "9.5": {
- "title": "rebroadcasts",
- "id": 88,
- "attr": {
- "style": {}
- }
- },
- "10.75": {
- "title": "likes",
- "id": 87,
- "attr": {
- "style": {}
- },
- "ideas": {
- "1": {
- "title": "count",
- "id": 90
- }
- }
- }
- },
- "attr": {}
- },
- "9": {
- "title": "content",
- "id": 114
- },
- "0.5": {
- "title": "actor",
- "id": 32,
- "ideas": {
- "1": {
- "title": "id:instagram:<id>",
- "id": 35
- },
- "2": {
- "title": "image",
- "id": 49,
- "ideas": {
- "1": {
- "title": "url",
- "id": 50
- }
- }
- },
- "0.5": {
- "title": "handle",
- "id": 36,
- "attr": {
- "position": [
- 63.5,
- -66,
- 1
- ],
- "style": {}
- }
- },
- "0.25": {
- "title": "extensions",
- "id": 47,
- "ideas": {
- "1": {
- "title": "screenName",
- "id": 48,
- "attr": {
- "style": {}
- }
- }
- },
- "attr": {
- "position": [
- 60.5,
- -126.5,
- 2
- ],
- "style": {}
- }
- }
- }
- }
- }
- },
- "-1": {
- "title": "data",
- "id": 4,
- "ideas": {
- "1": {
- "title": "comments",
- "id": 5,
- "ideas": {
- "1": {
- "title": "data",
- "id": 44
- }
- }
- },
- "3": {
- "title": "caption",
- "id": 7,
- "ideas": {}
- },
- "4": {
- "title": "links",
- "id": 9
- },
- "5": {
- "title": "link",
- "id": 10
- },
- "6": {
- "title": "user",
- "id": 11,
- "ideas": {
- "1": {
- "title": "username",
- "id": 33
- },
- "2": {
- "title": "profile_picture",
- "id": 34
- },
- "3": {
- "title": "id",
- "id": 37
- }
- }
- },
- "9": {
- "title": "type",
- "id": 21
- },
- "12": {
- "title": "videos",
- "id": 41,
- "ideas": {
- "1": {
- "title": "low_resolution",
- "id": 42,
- "ideas": {
- "1": {
- "title": "url",
- "id": 67
- },
- "2": {
- "title": "height",
- "id": 68
- },
- "3": {
- "title": "width",
- "id": 69
- }
- }
- },
- "2": {
- "title": "standard_resolution",
- "id": 43,
- "ideas": {
- "1": {
- "title": "height",
- "id": 70
- },
- "2": {
- "title": "width",
- "id": 71
- },
- "3": {
- "title": "url",
- "id": 72
- }
- }
- }
- }
- },
- "13": {
- "title": "tags",
- "id": 45,
- "ideas": {}
- },
- "14": {
- "title": "users_in_photo",
- "id": 95,
- "ideas": {
- "1": {
- "title": "id",
- "id": 96,
- "ideas": {}
- },
- "2": {
- "title": "profile_picture",
- "id": 98
- },
- "3": {
- "title": "username",
- "id": 97,
- "attr": {
- "style": {}
- }
- }
- }
- },
- "11.5": {
- "title": "images",
- "id": 14,
- "ideas": {
- "1": {
- "title": "low_resolution",
- "id": 15,
- "ideas": {
- "1": {
- "title": "url",
- "id": 18
- },
- "2": {
- "title": "height",
- "id": 58
- },
- "3": {
- "title": "width",
- "id": 59
- }
- }
- },
- "2": {
- "title": "thumbnail",
- "id": 16,
- "ideas": {
- "1": {
- "title": "url",
- "id": 19
- },
- "2": {
- "title": "height",
- "id": 60
- },
- "3": {
- "title": "width",
- "id": 61
- }
- }
- },
- "3": {
- "title": "standard_resolution",
- "id": 17,
- "ideas": {
- "1": {
- "title": "url",
- "id": 20
- },
- "2": {
- "title": "height",
- "id": 73
- },
- "3": {
- "title": "width",
- "id": 74
- }
- }
- }
- }
- },
- "6.5": {
- "title": "id",
- "id": 23
- },
- "6.25": {
- "title": "created_time",
- "id": 12
- },
- "12.5": {
- "title": "likes",
- "id": 6,
- "ideas": {
- "1": {
- "title": "count",
- "id": 89
- }
- },
- "attr": {
- "style": {}
- }
- },
- "12.25": {
- "title": "location",
- "id": 24,
- "ideas": {
- "1": {
- "title": "latitude",
- "id": 83
- },
- "2": {
- "title": "longitude",
- "id": 84
- }
- },
- "attr": {
- "style": {}
- }
- }
- }
- }
- },
- "links": [
- {
- "ideaIdFrom": 23,
- "ideaIdTo": 26,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 33,
- "ideaIdTo": 36,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 37,
- "ideaIdTo": 35,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 12,
- "ideaIdTo": 25,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 21,
- "ideaIdTo": 38,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 33,
- "ideaIdTo": 48,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 50,
- "ideaIdTo": 34,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 83,
- "ideaIdTo": 82,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 84,
- "ideaIdTo": 82,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 45,
- "ideaIdTo": 76,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 89,
- "ideaIdTo": 90,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 91,
- "ideaIdTo": 4,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 40,
- "ideaIdTo": 10,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 97,
- "ideaIdTo": 94,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 96,
- "ideaIdTo": 93,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 64,
- "ideaIdTo": 20,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 65,
- "ideaIdTo": 73,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 66,
- "ideaIdTo": 74,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 66,
- "ideaIdTo": 71,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 65,
- "ideaIdTo": 70,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 64,
- "ideaIdTo": 72,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 55,
- "ideaIdTo": 19,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 56,
- "ideaIdTo": 60,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 61,
- "ideaIdTo": 57,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 103,
- "ideaIdTo": 18,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 104,
- "ideaIdTo": 58,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 59,
- "ideaIdTo": 105,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 110,
- "ideaIdTo": 67,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 111,
- "ideaIdTo": 68,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 112,
- "ideaIdTo": 69,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 7,
- "ideaIdTo": 114,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- },
- {
- "ideaIdFrom": 113,
- "ideaIdTo": 7,
- "attr": {
- "style": {
- "color": "#FF0000",
- "lineStyle": "dashed"
- }
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/metadata/instagram_to_activity_mapping.png b/streams-contrib/streams-provider-instagram/metadata/instagram_to_activity_mapping.png
deleted file mode 100644
index a3b7050..0000000
Binary files a/streams-contrib/streams-provider-instagram/metadata/instagram_to_activity_mapping.png and /dev/null differ
diff --git a/streams-contrib/streams-provider-instagram/pom.xml b/streams-contrib/streams-provider-instagram/pom.xml
deleted file mode 100644
index f1293cc..0000000
--- a/streams-contrib/streams-provider-instagram/pom.xml
+++ /dev/null
@@ -1,226 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-provider-instagram</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Instagram Provider</description>
-
- <properties>
- <skipITs>true</skipITs>
- <testDataBaseURl>http://streams.peoplepattern.com.s3.amazonaws.com/test-data/</testDataBaseURl>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-joda</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path-assert</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.juneau</groupId>
- <artifactId>juneau-rest-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-all</artifactId>
- <version>1.3</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>1.9.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.instagram.pojo</targetPackage>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <version>1.2.1</version>
- <executions>
- <execution>
- <id>download-it-data</id>
- <phase>pre-integration-test</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
- <url>${testDataBaseURl}/${project.artifactId}.zip</url>
- <unpack>true</unpack>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- <!--<md5>df65b5642f33676313ebe4d5b69a3fff</md5>-->
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <configuration>
- <skipTests>${skipITs}</skipTests>
- <!-- Run integration test suite rather than individual tests. -->
- <excludes>
- <exclude>**/*Test.java</exclude>
- <exclude>**/*Tests.java</exclude>
- </excludes>
- <includes>
- <exclude>**/*IT.java</exclude>
- <include>**/*ITs.java</include>
- </includes>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
- <version>${failsafe.plugin.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/Instagram.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/Instagram.java
deleted file mode 100644
index e153b5f..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/Instagram.java
+++ /dev/null
@@ -1,499 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.api;
-
-import org.apache.streams.instagram.config.InstagramConfiguration;
-import org.apache.streams.instagram.pojo.UserRecentMediaRequest;
-import org.apache.streams.instagram.provider.InstagramProviderUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.juneau.JodaDateSwap;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.juneau.json.JsonParser;
-import org.apache.juneau.json.JsonSerializer;
-import org.apache.juneau.rest.client.RestCall;
-import org.apache.juneau.rest.client.RestCallException;
-import org.apache.juneau.rest.client.RestClient;
-import org.apache.juneau.rest.client.RestClientBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
-
-import static org.apache.http.entity.ContentType.APPLICATION_JSON;
-
-/**
- * Implementation of all instagram interfaces using juneau.
- */
-public class Instagram implements Media, Users {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(Instagram.class);
-
- private static Map<InstagramConfiguration, Instagram> INSTANCE_MAP = new ConcurrentHashMap<>();
-
- private InstagramConfiguration configuration;
-
- private ObjectMapper mapper;
-
- private CloseableHttpClient httpclient;
-
- private InstagramOAuthRequestSigner oauthSigner;
-
- private String rootUrl;
-
- RestClient restClient;
-
- private Instagram(InstagramConfiguration configuration) throws InstantiationException {
- this.configuration = configuration;
- this.rootUrl = InstagramProviderUtil.baseUrl(configuration);
- this.oauthSigner = new InstagramOAuthRequestSigner(configuration.getOauth());
- this.httpclient = HttpClientBuilder.create()
- .setDefaultRequestConfig(RequestConfig.custom()
- .setConnectionRequestTimeout(5000)
- .setConnectTimeout(5000)
- .setSocketTimeout(5000)
- .setCookieSpec("easy")
- .build()
- )
- .setMaxConnPerRoute(20)
- .setMaxConnTotal(100)
- .build();
- this.restClient = new RestClientBuilder()
- .rootUrl(rootUrl)
- .accept(APPLICATION_JSON.getMimeType())
- .httpClient(httpclient, true)
- .pooled()
- .parser(
- JsonParser.DEFAULT.builder()
- .ignoreUnknownBeanProperties(true)
- .pojoSwaps(JodaDateSwap.class)
- .build())
- .serializer(
- JsonSerializer.DEFAULT.builder()
- .pojoSwaps(JodaDateSwap.class)
- .build())
- .retryable(
- configuration.getRetryMax().intValue(),
- configuration.getRetrySleepMs(),
- new InstagramRetryHandler())
- .build();
- this.mapper = StreamsJacksonMapper.getInstance();
- }
-
- /**
- * getInstance of Instagram from InstagramConfiguration.
- *
- * @param configuration InstagramConfiguration
- * @return Instagram
- * @throws InstantiationException InstantiationException
- */
- public static Instagram getInstance(InstagramConfiguration configuration) throws InstantiationException {
- if (INSTANCE_MAP.containsKey(configuration) && INSTANCE_MAP.get(configuration) != null) {
- return INSTANCE_MAP.get(configuration);
- } else {
- Instagram instagram = new Instagram(configuration);
- INSTANCE_MAP.put(configuration, instagram);
- return INSTANCE_MAP.get(configuration);
- }
- }
-
- @Override
- public UserInfoResponse self() {
- try {
- // TODO: use juneau @Remotable
- // Users restUsers = restClient.getRemoteableProxy("/users", Users.class);
- // UserInfoResponse result = restUsers.lookupUser(parameters);
- // return result;
- URIBuilder uriBuilder = new URIBuilder()
- .setPath("/users/self/");
- uriBuilder.addParameter("access_token", configuration.getOauth().getAccessToken());
- String sig = oauthSigner.generateSignature(uriBuilder.build().toString());
- uriBuilder.addParameter("sig", sig);
- RestCall restCall = restClient.doGet(uriBuilder.build().toString());
- try {
- String restResponseEntity = restCall
- .getResponseAsString();
- UserInfoResponse result = mapper.readValue(restResponseEntity, UserInfoResponse.class);
- return result;
- } catch (RestCallException ex) {
- LOGGER.warn("RestCallException", ex);
- }
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- } catch (URISyntaxException ex) {
- LOGGER.warn("URISyntaxException", ex);
- } catch (Exception ex) {
- LOGGER.warn("Exception", ex);
- }
- return null;
- }
-
- @Override
- public UserInfoResponse lookupUser(String user_id) {
- try {
- // TODO: use juneau @Remotable
- // Users restUsers = restClient.getRemoteableProxy("/users", Users.class);
- // UserInfoResponse result = restUsers.lookupUser(parameters);
- // return result;
- URIBuilder uriBuilder = new URIBuilder()
- .setPath("/users/" + user_id);
- uriBuilder.addParameter("access_token", configuration.getOauth().getAccessToken());
- String sig = oauthSigner.generateSignature(uriBuilder.build().toString());
- uriBuilder.addParameter("sig", sig);
- RestCall restCall = restClient.doGet(uriBuilder.build().toString());
- try {
- String restResponseEntity = restCall
- .getResponseAsString();
- UserInfoResponse result = mapper.readValue(restResponseEntity, UserInfoResponse.class);
- return result;
- } catch (RestCallException ex) {
- LOGGER.warn("RestCallException", ex);
- }
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- } catch (URISyntaxException ex) {
- LOGGER.warn("URISyntaxException", ex);
- } catch (Exception ex) {
- LOGGER.warn("Exception", ex);
- }
- return null;
- }
-
- @Override
- public RecentMediaResponse selfMediaRecent(SelfRecentMediaRequest parameters) {
- try {
- // TODO: use juneau @Remotable
- // Users restUsers = restClient.getRemoteableProxy("/users", Users.class);
- // RecentMediaResponse result = restUsers.selfMediaRecent(parameters);
- // return result;
- URIBuilder uriBuilder = new URIBuilder()
- .setPath("/users/self/media/recent");
- uriBuilder.addParameter("access_token", configuration.getOauth().getAccessToken());
- if ( Objects.nonNull(parameters.getCount()) && StringUtils.isNotBlank(parameters.getCount().toString())) {
- uriBuilder.addParameter("count", parameters.getCount().toString());
- }
- if ( Objects.nonNull(parameters.getMaxId()) && StringUtils.isNotBlank(parameters.getMaxId().toString())) {
- uriBuilder.addParameter("max_id", parameters.getMaxId().toString());
- }
- if ( Objects.nonNull(parameters.getMinId()) && StringUtils.isNotBlank(parameters.getMinId().toString())) {
- uriBuilder.addParameter("min", parameters.getMinId().toString());
- }
- String sig = oauthSigner.generateSignature(uriBuilder.build().toString());
- uriBuilder.addParameter("sig", sig);
- RestCall restCall = restClient.doGet(uriBuilder.build().toString());
- try {
- String restResponseEntity = restCall
- .getResponseAsString();
- RecentMediaResponse result = mapper.readValue(restResponseEntity, RecentMediaResponse.class);
- return result;
- } catch (RestCallException ex) {
- LOGGER.warn("RestCallException", ex);
- }
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- } catch (URISyntaxException ex) {
- LOGGER.warn("URISyntaxException", ex);
- } catch (Exception ex) {
- LOGGER.warn("Exception", ex);
- }
- return null;
- }
-
- @Override
- public RecentMediaResponse userMediaRecent(UserRecentMediaRequest parameters) {
- try {
- // TODO: use juneau @Remotable
- // Users restUsers = restClient.getRemoteableProxy("/users", Users.class);
- // RecentMediaResponse result = restUsers.userMediaRecent(parameters);
- // return result;
- URIBuilder uriBuilder = new URIBuilder()
- .setPath("/users/" + parameters.getUserId() + "/media/recent");
- uriBuilder.addParameter("access_token", configuration.getOauth().getAccessToken());
- if ( Objects.nonNull(parameters.getCount()) && StringUtils.isNotBlank(parameters.getCount().toString())) {
- uriBuilder.addParameter("count", parameters.getCount().toString());
- }
- if ( Objects.nonNull(parameters.getMaxId()) && StringUtils.isNotBlank(parameters.getMaxId().toString())) {
- uriBuilder.addParameter("max_id", parameters.getMaxId().toString());
- }
- if ( Objects.nonNull(parameters.getMinId()) && StringUtils.isNotBlank(parameters.getMinId().toString())) {
- uriBuilder.addParameter("min", parameters.getMinId().toString());
- }
- String sig = oauthSigner.generateSignature(uriBuilder.build().toString());
- uriBuilder.addParameter("sig", sig);
- RestCall restCall = restClient.doGet(uriBuilder.build().toString());
- try {
- String restResponseEntity = restCall
- .getResponseAsString();
- RecentMediaResponse result = mapper.readValue(restResponseEntity, RecentMediaResponse.class);
- return result;
- } catch (RestCallException ex) {
- LOGGER.warn("RestCallException", ex);
- }
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- } catch (URISyntaxException ex) {
- LOGGER.warn("URISyntaxException", ex);
- } catch (Exception ex) {
- LOGGER.warn("Exception", ex);
- }
- return null;
- }
-
- @Override
- public RecentMediaResponse selfMediaLiked(SelfLikedMediaRequest parameters) {
- try {
- // TODO: use juneau @Remotable
- // Users restUsers = restClient.getRemoteableProxy("/users", Users.class);
- // RecentMediaResponse result = restUsers.selfMediaLiked(parameters);
- // return result;
- URIBuilder uriBuilder = new URIBuilder()
- .setPath("/users/self/media/liked");
- uriBuilder.addParameter("access_token", configuration.getOauth().getAccessToken());
- if ( Objects.nonNull(parameters.getCount()) && StringUtils.isNotBlank(parameters.getCount().toString())) {
- uriBuilder.addParameter("count", parameters.getCount().toString());
- }
- if ( Objects.nonNull(parameters.getMaxLikeId()) && StringUtils.isNotBlank(parameters.getMaxLikeId().toString())) {
- uriBuilder.addParameter("max_like_id", parameters.getMaxLikeId().toString());
- }
- String sig = oauthSigner.generateSignature(uriBuilder.build().toString());
- uriBuilder.addParameter("sig", sig);
- RestCall restCall = restClient.doGet(uriBuilder.build().toString());
- try {
- String restResponseEntity = restCall
- .getResponseAsString();
- RecentMediaResponse result = mapper.readValue(restResponseEntity, RecentMediaResponse.class);
- return result;
- } catch (RestCallException ex) {
- LOGGER.warn("RestCallException", ex);
- }
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- } catch (URISyntaxException ex) {
- LOGGER.warn("URISyntaxException", ex);
- } catch (Exception ex) {
- LOGGER.warn("Exception", ex);
- }
- return null;
- }
-
- @Override
- public SearchUsersResponse searchUser(SearchUsersRequest parameters) {
- try {
- // TODO: use juneau @Remotable
- // Users restUsers = restClient.getRemoteableProxy("/users", Users.class);
- // SearchUsersResponse result = restUsers.searchUser(parameters);
- // return result;
- URIBuilder uriBuilder = new URIBuilder()
- .setPath("/users/search");
- uriBuilder.addParameter("access_token", configuration.getOauth().getAccessToken());
- if ( Objects.nonNull(parameters.getCount()) && StringUtils.isNotBlank(parameters.getCount().toString())) {
- uriBuilder.addParameter("count", parameters.getCount().toString());
- }
- if ( Objects.nonNull(parameters.getQ()) && StringUtils.isNotBlank(parameters.getQ().toString())) {
- uriBuilder.addParameter("q", parameters.getQ().toString());
- }
- String sig = oauthSigner.generateSignature(uriBuilder.build().toString());
- uriBuilder.addParameter("sig", sig);
- RestCall restCall = restClient.doGet(uriBuilder.build().toString());
- try {
- String restResponseEntity = restCall
- .getResponseAsString();
- SearchUsersResponse result = mapper.readValue(restResponseEntity, SearchUsersResponse.class);
- return result;
- } catch (RestCallException ex) {
- LOGGER.warn("RestCallException", ex);
- }
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- } catch (URISyntaxException ex) {
- LOGGER.warn("URISyntaxException", ex);
- } catch (Exception ex) {
- LOGGER.warn("Exception", ex);
- }
- return null;
- }
-
- @Override
- public CommentsResponse comments(String media_id) {
- try {
- // TODO: use juneau @Remotable
- // Media restMedia = restClient.getRemoteableProxy("/media", Media.class);
- // CommentsResponse result = restMedia.comments(media_id);
- // return result;
- URIBuilder uriBuilder = new URIBuilder()
- .setPath("/media/" + media_id + "/comments");
- uriBuilder.addParameter("access_token", configuration.getOauth().getAccessToken());
- uriBuilder.addParameter("media_id", media_id);
- String sig = oauthSigner.generateSignature(uriBuilder.build().toString());
- uriBuilder.addParameter("sig", sig);
- RestCall restCall = restClient.doGet(uriBuilder.build().toString());
- try {
- String restResponseEntity = restCall
- .getResponseAsString();
- CommentsResponse result = mapper.readValue(restResponseEntity, CommentsResponse.class);
- return result;
- } catch (RestCallException ex) {
- LOGGER.warn("RestCallException", ex);
- }
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- } catch (URISyntaxException ex) {
- LOGGER.warn("URISyntaxException", ex);
- } catch (Exception ex) {
- LOGGER.warn("Exception", ex);
- }
- return null;
- }
-
- @Override
- public UsersInfoResponse likes(String media_id) {
- try {
- // TODO: use juneau @Remotable
- // Media restMedia = restClient.getRemoteableProxy("/media", Media.class);
- // UsersInfoResponse result = restMedia.likes(media_id);
- // return result;
- URIBuilder uriBuilder = new URIBuilder()
- .setPath("/media/" + media_id + "/likes");
- uriBuilder.addParameter("access_token", configuration.getOauth().getAccessToken());
- String sig = oauthSigner.generateSignature(uriBuilder.build().toString());
- uriBuilder.addParameter("sig", sig);
- RestCall restCall = restClient.doGet(uriBuilder.build().toString());
- try {
- String restResponseEntity = restCall
- .getResponseAsString();
- UsersInfoResponse result = mapper.readValue(restResponseEntity, UsersInfoResponse.class);
- return result;
- } catch (RestCallException ex) {
- LOGGER.warn("RestCallException", ex);
- }
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- } catch (URISyntaxException ex) {
- LOGGER.warn("URISyntaxException", ex);
- } catch (Exception ex) {
- LOGGER.warn("Exception", ex);
- }
- return null;
- }
-
- @Override
- public MediaResponse lookupMedia(String media_id) {
- try {
- // TODO: use juneau @Remotable
- // Media restMedia = restClient.getRemoteableProxy("/media", Media.class);
- // MediaResponse result = restMedia.lookupMedia(media_id);
- // return result;
- URIBuilder uriBuilder = new URIBuilder()
- .setPath("/media/" + media_id);
- uriBuilder.addParameter("access_token", configuration.getOauth().getAccessToken());
- String sig = oauthSigner.generateSignature(uriBuilder.build().toString());
- uriBuilder.addParameter("sig", sig);
- RestCall restCall = restClient.doGet(uriBuilder.build().toString());
- try {
- String restResponseEntity = restCall
- .getResponseAsString();
- MediaResponse result = mapper.readValue(restResponseEntity, MediaResponse.class);
- return result;
- } catch (RestCallException ex) {
- LOGGER.warn("RestCallException", ex);
- }
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- } catch (URISyntaxException ex) {
- LOGGER.warn("URISyntaxException", ex);
- } catch (Exception ex) {
- LOGGER.warn("Exception", ex);
- }
- return null;
- }
-
- @Override
- public MediaResponse shortcode(String shortcode) {
- try {
- // TODO: use juneau @Remotable
-// Media restMedia = restClient.getRemoteableProxy("/media", Media.class);
-// MediaResponse result = restMedia.lookupMedia(media_id);
-// return result;
- URIBuilder uriBuilder = new URIBuilder()
- .setPath("/media/shortcode/" + shortcode);
- uriBuilder.addParameter("access_token", configuration.getOauth().getAccessToken());
- String sig = oauthSigner.generateSignature(uriBuilder.build().toString());
- uriBuilder.addParameter("sig", sig);
- RestCall restCall = restClient.doGet(uriBuilder.build().toString());
- try {
- String restResponseEntity = restCall
- .getResponseAsString();
- MediaResponse result = mapper.readValue(restResponseEntity, MediaResponse.class);
- return result;
- } catch (RestCallException ex) {
- LOGGER.warn("RestCallException", ex);
- }
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- } catch (URISyntaxException ex) {
- LOGGER.warn("URISyntaxException", ex);
- } catch (Exception ex) {
- LOGGER.warn("Exception", ex);
- }
- return null;
- }
-
- @Override
- public SearchMediaResponse searchMedia(SearchMediaRequest parameters) {
- try {
- // TODO: use juneau @Remotable
- // Media restMedia = restClient.getRemoteableProxy("/media", Media.class);
- // SearchMediaResponse result = restMedia.lookupMedia(media_id);
- // return result;
- URIBuilder uriBuilder = new URIBuilder()
- .setPath("/media/search");
- uriBuilder.addParameter("access_token", configuration.getOauth().getAccessToken());
- uriBuilder.addParameter("distance", parameters.getDistance().toString());
- uriBuilder.addParameter("lat", parameters.getLat().toString());
- uriBuilder.addParameter("lng", parameters.getLng().toString());
- String sig = oauthSigner.generateSignature(uriBuilder.build().toString());
- uriBuilder.addParameter("sig", sig);
- RestCall restCall = restClient.doGet(uriBuilder.build().toString());
- try {
- String restResponseEntity = restCall
- .getResponseAsString();
- SearchMediaResponse result = mapper.readValue(restResponseEntity, SearchMediaResponse.class);
- return result;
- } catch (RestCallException ex) {
- LOGGER.warn("RestCallException", ex);
- }
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- } catch (URISyntaxException ex) {
- LOGGER.warn("URISyntaxException", ex);
- } catch (Exception ex) {
- LOGGER.warn("Exception", ex);
- }
- return null;
- }
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/InstagramOAuthRequestSigner.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/InstagramOAuthRequestSigner.java
deleted file mode 100644
index 02e5438..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/InstagramOAuthRequestSigner.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.api;
-
-import org.apache.streams.instagram.config.InstagramOAuthConfiguration;
-
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.HttpException;
-import org.apache.http.HttpRequest;
-import org.apache.http.HttpRequestInterceptor;
-import org.apache.http.protocol.HttpContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import java.security.GeneralSecurityException;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedSet;
-import java.util.StringJoiner;
-import java.util.TreeSet;
-import javax.crypto.Mac;
-import javax.crypto.spec.SecretKeySpec;
-
-/**
- * Handles request signing to api.instagram.com.
- *
- * @see <a href="https://www.instagram.com/developer/secure-api-requests/">https://www.instagram.com/developer/secure-api-requests/</a>
- */
-public class InstagramOAuthRequestSigner {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramOAuthRequestSigner.class);
-
- private static final String oauth_signature_encoding = "UTF-8";
- private static final String oauth_signature_method = "HmacSHA256";
-
- InstagramOAuthConfiguration oAuthConfiguration;
-
- public InstagramOAuthRequestSigner(InstagramOAuthConfiguration oauth) {
- this.oAuthConfiguration = oauth;
- }
-
- /**
- * generateSignature.
- * @param uri uri
- * @return String
- */
- public String generateSignature(String uri) {
-
- String request_path = uri.substring(0, uri.indexOf('?'));
- String request_param_line = uri.substring(uri.indexOf('?') + 1);
- String[] request_params = URLDecoder.decode(request_param_line).split("&");
-
- Map<String,String> oauthParamMap = new HashMap<>();
- oauthParamMap.put("access_token", oAuthConfiguration.getAccessToken());
-
- Map<String,String> allParamsMap = new HashMap<>(oauthParamMap);
-
- for ( String request_param : request_params ) {
- String key = request_param.substring(0, request_param.indexOf('='));
- String value = request_param.substring(request_param.indexOf('=') + 1, request_param.length());
- allParamsMap.put(key, value);
- }
-
- String endpoint = request_path;
-
- // the sig string to sign in the form "endpoint|key1=value1|key2=value2|...."
- String signature_base_string = generateSignatureBaseString(endpoint, allParamsMap);
-
- String oauth_signature;
- try {
- oauth_signature = computeSignature(signature_base_string, oAuthConfiguration.getClientSecret());
- } catch (GeneralSecurityException ex) {
- LOGGER.warn("GeneralSecurityException", ex);
- return null;
- } catch (UnsupportedEncodingException ex) {
- LOGGER.warn("UnsupportedEncodingException", ex);
- return null;
- }
-
- return oauth_signature;
- }
-
- /**
- * generateSignatureBaseString.
- * @param endpoint endpoint
- * @param allParamsMap allParamsMap
- * @return String
- */
- public static String generateSignatureBaseString(String endpoint, Map<String, String> allParamsMap) {
-
- SortedSet<String> sortedKeys = new TreeSet<>(allParamsMap.keySet());
-
- StringJoiner stringJoiner = new StringJoiner("|");
- stringJoiner.add(endpoint);
- for ( String key : sortedKeys ) {
- stringJoiner.add(key + "=" + allParamsMap.get(key));
- }
-
- return stringJoiner.toString();
- }
-
- /**
- * computeSignature.
- * @param signature_base_string Signature Base String
- * @param clientSecret Client Secret
- * @return String
- * @throws NoSuchAlgorithmException
- * @throws InvalidKeyException
- * @throws UnsupportedEncodingException
- */
- public static String computeSignature(String signature_base_string, String clientSecret) throws NoSuchAlgorithmException, InvalidKeyException, UnsupportedEncodingException {
- SecretKeySpec keySpec = new SecretKeySpec(clientSecret.getBytes(oauth_signature_encoding), oauth_signature_method);
- Mac mac = Mac.getInstance(oauth_signature_method);
- mac.init(keySpec);
- byte[] result = mac.doFinal(signature_base_string.getBytes(oauth_signature_encoding));
- return Hex.encodeHexString(result);
- }
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/InstagramRetryHandler.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/InstagramRetryHandler.java
deleted file mode 100644
index 51cd395..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/InstagramRetryHandler.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.api;
-
-import org.apache.streams.util.api.requests.backoff.AbstractBackOffStrategy;
-
-import org.apache.http.HttpResponse;
-import org.apache.juneau.rest.client.RetryOn;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Handle expected and unexpected exceptions.
- */
-public class InstagramRetryHandler extends RetryOn {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramRetryHandler.class);
-
- private static AbstractBackOffStrategy backoff_strategy;
-
- protected boolean onResponse(HttpResponse response) {
- LOGGER.debug(response.toString());
- switch(response.getStatusLine().getStatusCode()) {
- case 200: // Response.Status.OK
- case 304: // Response.Status.NOT_MODIFIED
- case 400: // Response.Status.BAD_REQUEST
- return false;
- case 401: // Response.Status.UNAUTHORIZED
- return true;
- case 403: // Response.Status.FORBIDDEN
- case 404: // Response.Status.NOT_FOUND
- case 406: // Response.Status.NOT_ACCEPTABLE
- case 410: // Response.Status.GONE
- return false;
- case 500: // Response.Status.INTERNAL_SERVER_ERROR
- case 502: // Bad Gateway
- case 503: // Response.Status.SERVICE_UNAVAILABLE
- case 504: // Gateway Timeout
- return true;
- default:
- return false;
- }
- }
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/Media.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/Media.java
deleted file mode 100644
index 0355f2b..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/Media.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.api;
-
-/**
- * Media Endpoints.
- * Comments Endpoints.
- * Like Endpoints.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/media/">https://www.instagram.com/developer/endpoints/media/</a>
- * @see <a href="https://www.instagram.com/developer/endpoints/comments/">https://www.instagram.com/developer/endpoints/comments/</a>
- * @see <a href="https://www.instagram.com/developer/endpoints/likes/">https://www.instagram.com/developer/endpoints/likes/</a>
- */
-public interface Media {
-
- /**
- * Get a list of recent comments on a media object.
- * The public_content scope is required for media that does not belong to the owner of the access_token.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/comments/#get_media_comments">https://www.instagram.com/developer/endpoints/comments/#get_media_comments</a>
- * @param media_id media_id
- * @return CommentsResponse @link{CommentsResponse}
- */
- public CommentsResponse comments(String media_id);
-
- /**
- * Get a list of users who have liked this media.
- * The public_content scope is required for media that does not belong to the owner of the access_token.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/likes/#get_media_likes">https://www.instagram.com/developer/endpoints/likes/#get_media_likes</a>
- * @param media_id media_id
- * @return UsersInfoResponse @link{UsersInfoResponse}
- */
- public UsersInfoResponse likes(String media_id);
-
- /**
- * Get information about a media object. Use the type field to differentiate between image and video media in the response. You will also receive the user_has_liked field which tells you whether the owner of the access_token has liked this media.
- *
- * The public_content permission scope is required to get a media that does not belong to the owner of the access_token.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/media/#get_media">https://www.instagram.com/developer/endpoints/media/#get_media</a>
- * @param media_id media_id
- * @return MediaResponse @link{MediaResponse}
- */
- public MediaResponse lookupMedia(String media_id);
-
- /**
- * This endpoint returns the same response as GET /media/media-id.
- * A media object's shortcode can be found in its shortlink URL. An example shortlink is http://instagram.com/p/tsxp1hhQTG/. Its corresponding shortcode is tsxp1hhQTG.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/media/#get_media_by_shortcode">https://www.instagram.com/developer/endpoints/media/#get_media_by_shortcode</a>
- * @param shortcode shortcode
- * @return MediaResponse @link{MediaResponse}
- */
- public MediaResponse shortcode(String shortcode);
-
- /**
- * Search for recent media in a given area.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/media/#get_media_search">https://www.instagram.com/developer/endpoints/media/#get_media_search</a>
- * @param parameters @link{SearchMediaRequest}
- * @return SearchMediaResponse @link{SearchMediaResponse}
- */
- public SearchMediaResponse searchMedia(SearchMediaRequest parameters);
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/Users.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/Users.java
deleted file mode 100644
index 82e2305..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/api/Users.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.api;
-
-import org.apache.streams.instagram.pojo.UserInfo;
-import org.apache.streams.instagram.pojo.UserRecentMediaRequest;
-
-/**
- * User Endpoints.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/users/">https://www.instagram.com/developer/endpoints/users/</a>
- */
-public interface Users {
-
- /**
- * Get information about the owner of the access_token.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/users/#get_users_self">https://www.instagram.com/developer/endpoints/users/#get_users_self</a>
- * @return UserInfoResponse @link{org.apache.streams.instagram.api.UserInfoResponse}
- */
- public UserInfoResponse self();
-
- /**
- * Get information about a user.
- * The public_content scope is required if the user is not the owner of the access_token.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/users/#get_users">https://www.instagram.com/developer/endpoints/users/#get_users</a>
- * @param user_id user_id
- * @return UserInfoResponse @link{org.apache.streams.instagram.api.UserInfoResponse}
- */
- public UserInfoResponse lookupUser(String user_id);
-
- /**
- * Get the most recent media published by the owner of the access_token.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/users/#get_users_media_recent_self">https://www.instagram.com/developer/endpoints/users/#get_users_media_recent_self</a>
- * @param parameters @link{org.apache.streams.instagram.api.SelfRecentMediaRequest}
- * @return RecentMediaResponse @link{org.apache.streams.instagram.api.RecentMediaResponse}
- */
- public RecentMediaResponse selfMediaRecent(SelfRecentMediaRequest parameters);
-
- /**
- * Get the most recent media published by a user.
- * The public_content scope is required if the user is not the owner of the access_token.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/users/#get_users_media_recent">https://www.instagram.com/developer/endpoints/users/#get_users_media_recent</a>
- * @param parameters @link{org.apache.streams.instagram.api.UserRecentMediaRequest}
- * @return RecentMediaResponse @link{org.apache.streams.instagram.api.RecentMediaResponse}
- */
- public RecentMediaResponse userMediaRecent(UserRecentMediaRequest parameters);
-
- /**
- * Get the list of recent media liked by the owner of the access_token.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/users/#get_users_feed_liked">https://www.instagram.com/developer/endpoints/users/#get_users_feed_liked</a>
- * @param parameters @link{org.apache.streams.instagram.api.SelfLikedMediaRequest}
- * @return RecentMediaResponse @link{org.apache.streams.instagram.api.RecentMediaResponse}
- */
- public RecentMediaResponse selfMediaLiked(SelfLikedMediaRequest parameters);
-
- /**
- * Get a list of users matching the query.
- *
- * @see <a href="https://www.instagram.com/developer/endpoints/users/#get_users_search">https://www.instagram.com/developer/endpoints/users/#get_users_search</a>
- * @param parameters @link{org.apache.streams.instagram.api.SearchUsersRequest}
- * @return SearchUsersResponse @link{org.apache.streams.instagram.api.SearchUsersResponse}
- */
- public SearchUsersResponse searchUser(SearchUsersRequest parameters);
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/processor/InstagramTypeConverter.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/processor/InstagramTypeConverter.java
deleted file mode 100644
index b20d7bc..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/processor/InstagramTypeConverter.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.instagram.pojo.Media;
-import org.apache.streams.instagram.pojo.UserInfo;
-import org.apache.streams.instagram.serializer.InstagramMediaFeedDataConverter;
-import org.apache.streams.instagram.serializer.InstagramUserInfoDataConverter;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * This is deprecated - use ActivityConverterProcessor or ActivityObjectConverterProcessor.
- */
-@Deprecated
-public class InstagramTypeConverter implements StreamsProcessor {
-
- public static final String STREAMS_ID = "InstagramTypeConverter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramTypeConverter.class);
-
- private InstagramMediaFeedDataConverter mediaFeedDataConverter;
- private InstagramUserInfoDataConverter userInfoDataConverter;
-
- public static final String TERMINATE = "TERMINATE";
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
-
- StreamsDatum result = null;
-
- try {
- Object item = entry.getDocument();
-
- LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
- if (item instanceof Media) {
-
- //We don't need to use the mapper, since we have a process to convert between
- //MediaFeedData objects and Activity objects already
- List<Activity> activity = mediaFeedDataConverter.toActivityList((Media)item);
-
- if ( activity.size() > 0 ) {
- result = new StreamsDatum(activity);
- }
-
- } else if (item instanceof UserInfo) {
-
- ActivityObject activityObject = userInfoDataConverter.toActivityObject((UserInfo)item);
-
- if ( activityObject != null ) {
- result = new StreamsDatum(activityObject);
- }
-
- }
-
- } catch (Exception ex) {
- ex.printStackTrace();
- LOGGER.error("Exception while converting item: {}", ex.getMessage());
- }
-
- if ( result != null ) {
- return Stream.of(result).collect(Collectors.toList());
- } else {
- return new ArrayList<>();
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
- mediaFeedDataConverter = new InstagramMediaFeedDataConverter();
- userInfoDataConverter = new InstagramUserInfoDataConverter();
- }
-
- @Override
- public void cleanUp() {
- //noop
- }
-
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramAbstractProvider.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramAbstractProvider.java
deleted file mode 100644
index ccc0777..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramAbstractProvider.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.instagram.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.instagram.api.Instagram;
-import org.apache.streams.instagram.config.InstagramConfiguration;
-import org.apache.streams.util.ComponentUtils;
-import org.apache.streams.util.SerializationUtil;
-
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Instagram {@link org.apache.streams.core.StreamsProvider} that provides Instagram data for a group of users
- */
-public abstract class InstagramAbstractProvider implements StreamsProvider {
-
- public static final String STREAMS_ID = "InstagramAbstractProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramAbstractProvider.class);
-
- private static final int MAX_BATCH_SIZE = 2000;
-
- protected InstagramConfiguration config;
-
- protected Instagram client;
-
- protected Queue<StreamsDatum> dataQueue;
- private ListeningExecutorService executorService;
-
-
- private List<ListenableFuture<Object>> futures = new ArrayList<>();
-
- private AtomicBoolean isCompleted;
-
- public InstagramAbstractProvider() {
- this.config = new ComponentConfigurator<>(InstagramConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("instagram"));
- }
-
- public InstagramAbstractProvider(InstagramConfiguration config) {
- this.config = SerializationUtil.cloneBySerialization(config);
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
- InstagramDataCollector dataCollector = getInstagramDataCollector();
- this.executorService = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor());
- ListenableFuture future = this.executorService.submit(dataCollector);
- this.futures.add(future);
- executorService.shutdown();
- }
-
- /**
- * Return the data collector to use to connect to instagram.
- * @return {@link InstagramDataCollector}
- */
- protected abstract InstagramDataCollector getInstagramDataCollector();
-
-
- @Override
- public StreamsResultSet readCurrent() {
- Queue<StreamsDatum> batch = new ConcurrentLinkedQueue<>();
- int count = 0;
- while (!this.dataQueue.isEmpty() && count < MAX_BATCH_SIZE) {
- ComponentUtils.offerUntilSuccess(ComponentUtils.pollWhileNotEmpty(this.dataQueue), batch);
- ++count;
- }
- return new StreamsResultSet(batch);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.dataQueue = new ConcurrentLinkedQueue<>();
- this.isCompleted = new AtomicBoolean(false);
-
- Objects.requireNonNull(config);
- Objects.requireNonNull(config.getOauth());
- Objects.requireNonNull(config.getOauth().getClientId());
- Objects.requireNonNull(config.getOauth().getClientSecret());
- Objects.requireNonNull(config.getOauth().getAccessToken());
- Objects.requireNonNull(config.getThreadsPerProvider());
-
- try {
- client = Instagram.getInstance(this.config);
- } catch (InstantiationException ex) {
- LOGGER.error("InstantiationException", ex);
- }
-
- Objects.requireNonNull(client);
-
- }
-
- @Override
- public void cleanUp() {
- try {
- ComponentUtils.shutdownExecutor(this.executorService, 5, 5);
- } finally {
- this.executorService = null;
- }
- }
-
- @Override
- public boolean isRunning() {
- if (dataQueue.isEmpty() && executorService.isTerminated() && Futures.allAsList(futures).isDone()) {
- LOGGER.info("Completed");
- isCompleted.set(true);
- LOGGER.info("Exiting");
- }
- return !isCompleted.get();
- }
-
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramDataCollector.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramDataCollector.java
deleted file mode 100644
index 741ceca..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramDataCollector.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.instagram.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.instagram.api.Instagram;
-import org.apache.streams.instagram.config.InstagramConfiguration;
-import org.apache.streams.instagram.config.InstagramOAuthConfiguration;
-import org.apache.streams.util.ComponentUtils;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
-import org.apache.streams.util.oauth.tokens.tokenmanager.SimpleTokenManager;
-import org.apache.streams.util.oauth.tokens.tokenmanager.impl.BasicTokenManager;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.Queue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Executes on all of the Instagram requests to collect the Instagram data.
- * <p/>
- * If errors/exceptions occur when trying to gather data for a particular user, that user is skipped and the collector
- * move on to the next user. If a rate limit exception occurs it employs an exponential back off strategy.
- */
-public abstract class InstagramDataCollector<T> implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramDataCollector.class);
-
- protected Queue<StreamsDatum> dataQueue; //exposed for testing
- private InstagramConfiguration config;
- protected AtomicBoolean isCompleted;
- private SimpleTokenManager<InstagramOAuthConfiguration> tokenManger;
- protected int consecutiveErrorCount;
- protected BackOffStrategy backOffStrategy;
- private Instagram instagram;
-
- /**
- * InstagramDataCollector constructor.
- * @param queue Queue of StreamsDatum
- * @param config InstagramConfiguration
- */
- public InstagramDataCollector(Instagram instagram, Queue<StreamsDatum> queue, InstagramConfiguration config) {
- this.dataQueue = queue;
- this.config = config;
- this.isCompleted = new AtomicBoolean(false);
- this.tokenManger = new BasicTokenManager<InstagramOAuthConfiguration>();
- this.consecutiveErrorCount = 0;
- this.backOffStrategy = new ExponentialBackOffStrategy(2);
- this.instagram = instagram;
- }
-
-
- /**
- * If there are authorized tokens available, it sets a new token for the client and returns
- * the client. If there are no available tokens, it simply returns the client that was
- * initialized in the constructor with client id.
- * @return result
- */
- protected Instagram getNextInstagramClient() {
- // if (this.tokenManger.numAvailableTokens() > 0) {
- // this.instagram.setAccessToken(this.tokenManger.getNextAvailableToken());
- // }
- return this.instagram;
- }
-
- /**
- * Return the number of available tokens for this data collector.
- * @return numbeer of available tokens
- */
- protected int numAvailableTokens() {
- return this.tokenManger.numAvailableTokens();
- }
-
- /**
- * Queues the Instagram data to be output by the provider.
- * @param userData data to queue
- * @param userId user id who the data came from
- */
- protected void queueData(Collection<T> userData, String userId) {
- if (userData == null) {
- LOGGER.warn("User id, {}, returned a NULL data from instagram.", userId);
- } else {
- for (T data : userData) {
- ComponentUtils.offerUntilSuccess(convertToStreamsDatum(data), this.dataQueue);
- }
- }
- }
-
- /**
- * @return true when the collector has queued all of the available Instagram data for the provided users.
- */
- public boolean isCompleted() {
- return this.isCompleted.get();
- }
-
- /**
- * Takes an Instagram Object and sets it as the document of a streams datum and sets the id of the streams datum.
- * @param item item
- * @return StreamsDatum
- */
- protected abstract StreamsDatum convertToStreamsDatum(T item);
-
-
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramProviderUtil.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramProviderUtil.java
deleted file mode 100644
index 62b36c3..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramProviderUtil.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.provider;
-
-import org.apache.streams.instagram.config.InstagramConfiguration;
-
-/**
- * TwitterProviderUtil contains utilities for Twitter Providers.
- */
-public class InstagramProviderUtil {
-
- /**
- * baseUrl from TwitterConfiguration.
- * @param config TwitterConfiguration
- * @return baseUrl
- */
- public static String baseUrl(InstagramConfiguration config) {
-
- String baseUrl = new StringBuilder()
- .append(config.getProtocol())
- .append("://")
- .append(config.getHost())
- .append(":")
- .append(config.getPort())
- .append("/")
- .append(config.getVersion())
- .append("/")
- .toString();
-
- return baseUrl;
- }
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/recentmedia/InstagramRecentMediaCollector.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/recentmedia/InstagramRecentMediaCollector.java
deleted file mode 100644
index 06616b0..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/recentmedia/InstagramRecentMediaCollector.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.instagram.provider.recentmedia;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.instagram.api.Instagram;
-import org.apache.streams.instagram.api.RecentMediaResponse;
-import org.apache.streams.instagram.config.InstagramRecentMediaProviderConfiguration;
-import org.apache.streams.instagram.pojo.Media;
-import org.apache.streams.instagram.pojo.UserRecentMediaRequest;
-import org.apache.streams.instagram.provider.InstagramDataCollector;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Queue;
-
-/**
- * Executes on all of the Instagram requests to collect the media feed data.
- * <p/>
- * If errors/exceptions occur when trying to gather data for a particular user, that user is skipped and the collector
- * move on to the next user. If a rate limit exception occurs it employs an exponential back off strategy.
- */
-public class InstagramRecentMediaCollector extends InstagramDataCollector<Media> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramRecentMediaCollector.class);
- protected static final int MAX_ATTEMPTS = 5;
-
- private int consecutiveErrorCount;
-
- InstagramRecentMediaProviderConfiguration config;
-
- public InstagramRecentMediaCollector(Instagram instagram, Queue<StreamsDatum> queue, InstagramRecentMediaProviderConfiguration config) {
- super(instagram, queue, config);
- this.config = config;
- }
-
- @Override
- public void run() {
- for (String userId : this.config.getInfo()) {
- try {
- collectInstagramDataForUser(userId);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- } catch (Exception ex) {
- LOGGER.error("Exception thrown while polling for user, {}, skipping user.", userId);
- LOGGER.error("Exception thrown while polling for user : ", ex);
- }
- }
- this.isCompleted.set(true);
- }
-
- protected StreamsDatum convertToStreamsDatum(Media item) {
- return new StreamsDatum(item, item.getId());
- }
-
- /**
- * Pull Recement Media for a user and queues the resulting data.
- * @param userId userId
- * @throws Exception Exception
- */
- protected void collectInstagramDataForUser(String userId) throws Exception {
-
- int item_count = 0;
- int last_count = 0;
- int page_count = 0;
-
- UserRecentMediaRequest request = (UserRecentMediaRequest) new UserRecentMediaRequest()
- .withUserId(userId)
- .withMinId(null)
- .withMaxId(0L)
- .withCount(33L);
- RecentMediaResponse response;
- do {
- response = getNextInstagramClient().userMediaRecent(request);
- if ( response != null && response.getData() != null) {
- last_count = response.getData().size();
- List<Media> data = new LinkedList<>();
- data.addAll(response.getData());
- super.queueData(data, userId);
- item_count += last_count;
- }
- page_count++;
- if ( shouldContinuePulling(response) ) {
- request.setMaxId(new Long(response.getPagination().getNextMaxId()));
- }
- }
- while (shouldContinuePulling(response));
-
- LOGGER.info("item_count: {} last_count: {} page_count: {} ", item_count, last_count, page_count);
-
- }
-
- private boolean shouldContinuePulling(RecentMediaResponse response) {
- if ( response != null
- || response.getData() != null
- || response.getData().size() > 0
- || response.getPagination() != null
- || StringUtils.isBlank(response.getPagination().getNextMaxId())) {
- return false;
- } else {
- return true;
- }
- }
-
-
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/recentmedia/InstagramRecentMediaProvider.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/recentmedia/InstagramRecentMediaProvider.java
deleted file mode 100644
index c8bf378..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/recentmedia/InstagramRecentMediaProvider.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.instagram.provider.recentmedia;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.instagram.config.InstagramConfiguration;
-import org.apache.streams.instagram.config.InstagramRecentMediaProviderConfiguration;
-import org.apache.streams.instagram.provider.InstagramAbstractProvider;
-import org.apache.streams.instagram.provider.InstagramDataCollector;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.Iterator;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Instagram {@link org.apache.streams.core.StreamsProvider} that provides the recent media data for a group of users.
- *
- * <p/>
- * Retrieve recent posts from a list of user ids or names.
- *
- */
-public class InstagramRecentMediaProvider extends InstagramAbstractProvider {
-
- public static final String STREAMS_ID = "InstagramRecentMediaProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramRecentMediaProvider.class);
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- InstagramRecentMediaProviderConfiguration config;
-
- public InstagramRecentMediaProvider(InstagramRecentMediaProviderConfiguration config) {
- super(config);
- this.config = config;
- }
-
- @Override
- protected InstagramDataCollector getInstagramDataCollector() {
- return new InstagramRecentMediaCollector(client, super.dataQueue, config);
- }
-
- /**
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * instagram.clientKey
- * instagram.usersInfo.authorizedTokens
- * instagram.usersInfo.users
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java \
- * -Dexec.mainClass=org.apache.streams.instagram.provider.recentmedia.InstagramRecentMediaProvider \
- * -Dexec.args="application.conf media.json"
- *
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = conf.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- InstagramRecentMediaProviderConfiguration config = new ComponentConfigurator<>(InstagramRecentMediaProviderConfiguration.class).detectConfiguration(typesafe, "instagram");
- InstagramRecentMediaProvider provider = new InstagramRecentMediaProvider(config);
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
- while (iterator.hasNext()) {
- StreamsDatum datum = iterator.next();
- String json;
- try {
- json = MAPPER.writeValueAsString(datum.getDocument());
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoCollector.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoCollector.java
deleted file mode 100644
index 1985435..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoCollector.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.instagram.provider.userinfo;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.instagram.api.Instagram;
-import org.apache.streams.instagram.api.UserInfoResponse;
-import org.apache.streams.instagram.config.InstagramConfiguration;
-import org.apache.streams.instagram.config.InstagramUserInfoProviderConfiguration;
-import org.apache.streams.instagram.pojo.UserInfo;
-import org.apache.streams.instagram.provider.InstagramDataCollector;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Queue;
-
-/**
- * InstagramDataCollector that pulls UserInfoData from Instagram
- * @see org.apache.streams.instagram.provider.InstagramDataCollector
- */
-public class InstagramUserInfoCollector extends InstagramDataCollector<UserInfo> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoCollector.class);
- protected static final int MAX_ATTEMPTS = 5;
-
- private int consecutiveErrorCount;
-
- InstagramUserInfoProviderConfiguration config;
-
- public InstagramUserInfoCollector(Instagram instagram, Queue<StreamsDatum> dataQueue, InstagramUserInfoProviderConfiguration config) {
- super(instagram, dataQueue, config);
- this.config = config;
- }
-
- @Override
- public void run() {
- for (String userId : this.config.getInfo()) {
- try {
- collectInstagramDataForUser(userId);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- } catch (Exception ex) {
- LOGGER.error("Exception thrown while polling for user, {}, skipping user.", userId);
- LOGGER.error("Exception thrown while polling for user : ", ex);
- }
- }
- this.isCompleted.set(true);
- }
-
- protected void collectInstagramDataForUser(String userId) throws Exception {
- UserInfoResponse userInfoResponse = null;
- try {
- userInfoResponse = getNextInstagramClient().lookupUser(userId);
- } catch (Exception ex) {
- LOGGER.error("Expection while polling instagram : {}", ex);
- }
- if ( userInfoResponse != null && userInfoResponse.getData() != null) {
- List<UserInfo> data = new LinkedList<>();
- data.add(userInfoResponse.getData());
- super.queueData(data, userId);
- }
- }
-
- @Override
- protected StreamsDatum convertToStreamsDatum(UserInfo item) {
- return new StreamsDatum(item, item.getId());
- }
-
-
-
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoProvider.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoProvider.java
deleted file mode 100644
index c74e911..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoProvider.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.instagram.provider.userinfo;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.instagram.config.InstagramConfiguration;
-import org.apache.streams.instagram.config.InstagramUserInfoProviderConfiguration;
-import org.apache.streams.instagram.provider.InstagramAbstractProvider;
-import org.apache.streams.instagram.provider.InstagramDataCollector;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.Iterator;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Instagram provider that pulls UserInfoData from Instagram.
- *
- * <p/>
- * Retrieve latest user details from a list of user ids or names.
- */
-public class InstagramUserInfoProvider extends InstagramAbstractProvider {
-
- public static final String STREAMS_ID = "InstagramUserInfoProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoProvider.class);
-
- private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- InstagramUserInfoProviderConfiguration config;
-
- public InstagramUserInfoProvider(InstagramUserInfoProviderConfiguration config) {
- super(config);
- this.config = config;
- }
-
- @Override
- protected InstagramDataCollector getInstagramDataCollector() {
- return new InstagramUserInfoCollector(client, super.dataQueue, config);
- }
-
- /**
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * instagram.clientKey
- * instagram.usersInfo.authorizedTokens
- * instagram.usersInfo.users
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java \
- * -Dexec.mainClass=org.apache.streams.instagram.provider.userinfo.InstagramUserInfoProvider \
- * -Dexec.args="application.conf userinfo.json"
- *
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = conf.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- InstagramUserInfoProviderConfiguration config = new ComponentConfigurator<>(InstagramUserInfoProviderConfiguration.class)
- .detectConfiguration(typesafe, "instagram");
- InstagramUserInfoProvider provider = new InstagramUserInfoProvider(config);
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
- while (iterator.hasNext()) {
- StreamsDatum datum = iterator.next();
- String json;
- try {
- json = MAPPER.writeValueAsString(datum.getDocument());
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramMediaFeedDataConverter.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramMediaFeedDataConverter.java
deleted file mode 100644
index 946a4e4..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramMediaFeedDataConverter.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.serializer;
-
-import org.apache.streams.data.ActivityConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.instagram.pojo.Media;
-import org.apache.streams.pojo.json.Activity;
-
-import org.apache.commons.lang.NotImplementedException;
-
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.List;
-
-import static org.apache.streams.instagram.serializer.util.InstagramActivityUtil.updateActivity;
-
-public class InstagramMediaFeedDataConverter implements ActivityConverter<Media>, Serializable {
-
- public static Class requiredClass = Media.class;
-
- public InstagramMediaFeedDataConverter() {
-
- }
-
- @Override
- public Class requiredClass() {
- return requiredClass;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public Media fromActivity(Activity deserialized) throws ActivityConversionException {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Media> fromActivityList(List<Activity> list) throws ActivityConversionException {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Activity> toActivityList(Media item) throws ActivityConversionException {
-
- Activity activity = new Activity();
-
- updateActivity(item, activity);
-
- return Collections.singletonList(activity);
- }
-
- @Override
- public List<Activity> toActivityList(List<Media> list) throws ActivityConversionException {
- throw new NotImplementedException();
- }
-
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramUserInfoDataConverter.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramUserInfoDataConverter.java
deleted file mode 100644
index 6c19167..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramUserInfoDataConverter.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.serializer;
-
-import org.apache.streams.data.ActivityObjectConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.instagram.pojo.UserInfo;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.pojo.json.Provider;
-
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * InstagramUserInfoDataConverter.
- */
-public class InstagramUserInfoDataConverter implements ActivityObjectConverter<UserInfo> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoDataConverter.class);
-
- private static final String STREAMS_ID_PREFIX = "id:instagram:";
- private static final String PROVIDER_ID = "id:provider:instagram";
- private static final String DISPLAY_NAME = "Instagram";
-
- @Override
- public Class requiredClass() {
- return UserInfo.class;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public UserInfo fromActivityObject(ActivityObject deserialized) throws ActivityConversionException {
- return null;
- }
-
- @Override
- public ActivityObject toActivityObject(UserInfo serialized) throws ActivityConversionException {
- ActivityObject activityObject = new ActivityObject();
- activityObject.setObjectType("page");
- Provider provider = new Provider();
- provider.setId(PROVIDER_ID);
- provider.setDisplayName(DISPLAY_NAME);
- activityObject.getAdditionalProperties().put("provider", provider);
- activityObject.setPublished(DateTime.now().withZone(DateTimeZone.UTC));
- Image image = new Image();
- image.setUrl(serialized.getProfilePicture());
- activityObject.setImage(image);
- activityObject.setId(STREAMS_ID_PREFIX + serialized.getId());
- activityObject.setSummary(serialized.getBio());
- activityObject.setAdditionalProperty("handle", serialized.getUsername());
- activityObject.setDisplayName(serialized.getFullName());
- activityObject.setUrl(serialized.getWebsite());
- Map<String, Object> extensions = new HashMap<>();
- activityObject.setAdditionalProperty("extensions", extensions);
- extensions.put("screenName", serialized.getUsername());
- extensions.put("posts", serialized.getCounts().getMedia());
- extensions.put("followers", serialized.getCounts().getFollowedBy());
- extensions.put("website", serialized.getWebsite());
- extensions.put("following", serialized.getCounts().getFollows());
- return activityObject;
- }
-
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
deleted file mode 100644
index 147a6e1..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java
+++ /dev/null
@@ -1,358 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.serializer.util;
-
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.instagram.pojo.Comment;
-import org.apache.streams.instagram.pojo.Comments;
-import org.apache.streams.instagram.pojo.Images;
-import org.apache.streams.instagram.pojo.Media;
-import org.apache.streams.instagram.pojo.UserInfo;
-import org.apache.streams.instagram.pojo.UserInfoCounts;
-import org.apache.streams.instagram.pojo.Videos;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.pojo.json.ImageParent;
-import org.apache.streams.pojo.json.Provider;
-
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Provides utilities for working with Activity objects within the context of Instagram.
- */
-public class InstagramActivityUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramActivityUtil.class);
- /**
- * Updates the given Activity object with the values from the item
- * @param item the object to use as the source
- * @param activity the target of the updates. Will receive all values from the tweet.
- * @throws ActivityConversionException ActivityConversionException
- */
-
- public static void updateActivity(Media item, Activity activity) throws ActivityConversionException {
- activity.setActor(buildActor(item));
- activity.setVerb("post");
-
- if (item.getCreatedTime() != null) {
- activity.setPublished(new DateTime(Long.parseLong(item.getCreatedTime()) * 1000));
- }
-
- activity.setId(formatId(activity.getVerb(),
- Optional.ofNullable(item.getId()).orElse(null)));
-
- activity.setProvider(getProvider());
- activity.setUrl(item.getLink());
- activity.setObject(buildActivityObject(item));
-
- if (item.getCaption() != null) {
- activity.setContent(item.getCaption().getText());
- }
-
- addInstagramExtensions(activity, item);
- }
-
- /**
- * Updates the given Activity object with the values from the item
- * @param item the object to use as the source
- * @param activity the target of the updates. Will receive all values from the tweet.
- * @throws ActivitySerializerException ActivitySerializerException
- */
- public static void updateActivity(UserInfo item, Activity activity) throws ActivitySerializerException {
- activity.setActor(buildActor(item));
- activity.setId(null);
- activity.setProvider(getProvider());
- }
-
- /**
- * Builds an Actor object given a UserInfo object.
- * @param item UserInfo item
- * @return Actor object
- */
- public static ActivityObject buildActor(UserInfo item) {
- ActivityObject actor = new ActivityObject();
-
- try {
- Image image = new Image();
- image.setUrl(item.getProfilePicture());
-
- UserInfoCounts counts = item.getCounts();
-
- Map<String, Object> extensions = new HashMap<>();
-
- extensions.put("followers", counts.getFollowedBy());
- extensions.put("follows", counts.getFollows());
- extensions.put("screenName", item.getUsername());
- extensions.put("posts", counts.getMedia());
-
- actor.setId(formatId(String.valueOf(item.getId())));
- actor.setImage(image);
- actor.setDisplayName(item.getFullName());
- actor.setSummary(item.getBio());
- actor.setUrl(item.getWebsite());
-
- actor.setAdditionalProperty("handle", item.getUsername());
- actor.setAdditionalProperty("extensions", extensions);
- } catch (Exception ex) {
- LOGGER.error("Exception trying to build actor object: {}", ex.getMessage());
- }
-
- return actor;
- }
-
- /**
- * Builds the actor.
- * @param item Media item
- * @return a valid ActivityObject
- */
- public static ActivityObject buildActor(Media item) {
- ActivityObject actor = new ActivityObject();
-
- try {
- Image image = new Image();
- image.setUrl(item.getUser().getProfilePicture());
-
- Map<String, Object> extensions = new HashMap<>();
- extensions.put("screenName", item.getUser().getUsername());
-
- actor.setDisplayName(item.getUser().getFullName());
- actor.setSummary(item.getUser().getBio());
- actor.setUrl(item.getUser().getWebsite());
-
- actor.setId(formatId(String.valueOf(item.getUser().getId())));
- actor.setImage(image);
- actor.setAdditionalProperty("extensions", extensions);
- actor.setAdditionalProperty("handle", item.getUser().getUsername());
- } catch (Exception ex) {
- LOGGER.error("Exception trying to build actor object: {}", ex.getMessage());
- }
-
- return actor;
- }
-
- /**
- * Builds the object.
- * @param item the item
- * @return a valid Activity Object
- */
- public static ActivityObject buildActivityObject(Media item) {
- ActivityObject actObj = new ActivityObject();
-
- actObj.setObjectType(item.getType().toString());
- actObj.setAttachments(buildActivityObjectAttachments(item));
-
- Image standardResolution = new Image();
- if (item.getType().equals("image") && item.getImages() != null) {
- ImageParent standardResolutionData = item.getImages().getStandardResolution();
- standardResolution.setHeight((long) standardResolutionData.getHeight());
- standardResolution.setWidth((long) standardResolutionData.getWidth());
- standardResolution.setUrl(standardResolutionData.getUrl());
- } else if (item.getType().equals("video") && item.getVideos() != null) {
- ImageParent standardResolutionData = item.getVideos().getStandardResolution();
- standardResolution.setHeight((long) standardResolutionData.getHeight());
- standardResolution.setWidth((long) standardResolutionData.getWidth());
- standardResolution.setUrl(standardResolutionData.getUrl());
- }
-
- actObj.setImage(standardResolution);
-
- return actObj;
- }
-
- /**
- * Builds all of the attachments associated with a Media object.
- *
- * @param item item
- * @return result
- */
- public static List<ActivityObject> buildActivityObjectAttachments(Media item) {
- List<ActivityObject> attachments = new ArrayList<>();
-
- addImageObjects(attachments, item);
- addVideoObjects(attachments, item);
-
- return attachments;
- }
-
- /**
- * Adds any image objects to the attachment field.
- * @param attachments attachments
- * @param item item
- */
- public static void addImageObjects(List<ActivityObject> attachments, Media item) {
- Images images = item.getImages();
-
- if (images != null) {
- try {
- ImageParent thumbnail = images.getThumbnail();
- ImageParent lowResolution = images.getLowResolution();
-
- ActivityObject thumbnailObject = new ActivityObject();
- Image thumbnailImage = new Image();
- thumbnailImage.setUrl(thumbnail.getUrl());
- thumbnailImage.setHeight((long) thumbnail.getHeight());
- thumbnailImage.setWidth((long) thumbnail.getWidth());
- thumbnailObject.setImage(thumbnailImage);
- thumbnailObject.setObjectType("image");
-
- ActivityObject lowResolutionObject = new ActivityObject();
- Image lowResolutionImage = new Image();
- lowResolutionImage.setUrl(lowResolution.getUrl());
- lowResolutionImage.setHeight((long) lowResolution.getHeight());
- lowResolutionImage.setWidth((long) lowResolution.getWidth());
- lowResolutionObject.setImage(lowResolutionImage);
- lowResolutionObject.setObjectType("image");
-
- attachments.add(thumbnailObject);
- attachments.add(lowResolutionObject);
- } catch (Exception ex) {
- LOGGER.error("Failed to add image objects: {}", ex.getMessage());
- }
- }
- }
-
- /**
- * Adds any video objects to the attachment field.
- * @param attachments attachments
- * @param item item
- */
- public static void addVideoObjects(List<ActivityObject> attachments, Media item) {
- Videos videos = item.getVideos();
-
- if (videos != null) {
- try {
- ImageParent lowResolutionVideo = videos.getLowResolution();
-
- ActivityObject lowResolutionVideoObject = new ActivityObject();
- Image lowResolutionVideoImage = new Image();
- lowResolutionVideoImage.setUrl(lowResolutionVideo.getUrl());
- lowResolutionVideoImage.setHeight((long) lowResolutionVideo.getHeight());
- lowResolutionVideoImage.setWidth((long) lowResolutionVideo.getWidth());
- lowResolutionVideoObject.setImage(lowResolutionVideoImage);
- lowResolutionVideoObject.setObjectType("video");
-
- attachments.add(lowResolutionVideoObject);
- } catch (Exception ex) {
- LOGGER.error("Failed to add video objects: {}", ex.getMessage());
- }
- }
- }
-
- /**
- * Gets the links from the Instagram event.
- * @param item the object to use as the source
- * @return a list of links corresponding to the expanded URL
- */
- public static List<String> getLinks(Media item) {
- return new ArrayList<>();
- }
-
- /**
- * Adds the location extension and populates with the instagram data.
- * @param activity the Activity object to update
- * @param item the object to use as the source
- */
- public static void addLocationExtension(Activity activity, Media item) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
-
- if (item.getLocation() != null) {
- Map<String, Object> coordinates = new HashMap<>();
- coordinates.put("type", "Point");
- coordinates.put("coordinates", "[" + item.getLocation().getLongitude() + "," + item.getLocation().getLatitude() + "]");
-
- extensions.put("coordinates", coordinates);
- }
- }
-
- /**
- * Gets the common instagram {@link Provider} object.
- * @return a provider object representing Instagram
- */
- public static Provider getProvider() {
- Provider provider = new Provider();
- provider.setId("id:providers:instagram");
- provider.setDisplayName("Instagram");
- return provider;
- }
-
- /**
- * Formats the ID to conform with the Apache Streams activity ID convention
- * @param idparts the parts of the ID to join
- * @return a valid Activity ID in format "id:instagram:part1:part2:...partN"
- */
- public static String formatId(String... idparts) {
- return String.join(":",
- Stream.concat(Arrays.stream(new String[]{"id:instagram"}), Arrays.stream(idparts)).collect(Collectors.toList()));
- }
-
- /**
- * Takes various parameters from the instagram object that are currently not part of the
- * activity schema and stores them in a generic extensions attribute.
- * @param activity Activity activity
- * @param item Media item
- */
- public static void addInstagramExtensions(Activity activity, Media item) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
-
- addLocationExtension(activity, item);
-
- if (item.getLikes() != null) {
- Map<String, Object> likes = new HashMap<>();
- likes.put("count", item.getLikes().getCount());
- extensions.put("likes", likes);
- }
-
- if (item.getComments() != null) {
- Map<String, Object> comments = new HashMap<>();
- comments.put("count", item.getComments().getCount());
- extensions.put("comments", comments);
- }
-
- extensions.put("hashtags", item.getTags());
-
- Comments comments = item.getComments();
- String commentsConcat = "";
-
- if (comments != null) {
- for (Comment comment : comments.getData()) {
- commentsConcat += " " + comment.getText();
- }
- }
- if (item.getCaption() != null) {
- commentsConcat += " " + item.getCaption().getText();
- }
-
- extensions.put("keywords", commentsConcat);
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/CommentsResponse.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/CommentsResponse.json
deleted file mode 100644
index e315871..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/CommentsResponse.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.CommentsResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "data": {
- "type": "array",
- "items": {
- "$ref": "../pojo/Comment.json"
- }
- },
- "meta": {
- "javaType": "org.apache.streams.instagram.api.Meta"
- },
- "pagination": {
- "javaType": "org.apache.streams.instagram.api.Pagination"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/MediaResponse.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/MediaResponse.json
deleted file mode 100644
index cbf4ebc..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/MediaResponse.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.MediaResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "data": {
- "$ref": "../pojo/Media.json"
- },
- "meta": {
- "javaType": "org.apache.streams.instagram.api.Meta"
- },
- "pagination": {
- "javaType": "org.apache.streams.instagram.api.Pagination"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/Meta.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/Meta.json
deleted file mode 100644
index 956f478..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/Meta.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.Meta",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "code": {
- "type": "integer"
- },
- "error_message": {
- "type": "string"
- },
- "error_type": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/Pagination.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/Pagination.json
deleted file mode 100644
index 315c416..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/Pagination.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.Pagination",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "next_url": {
- "type": "string"
- },
- "next_max_id": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/RecentMediaResponse.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/RecentMediaResponse.json
deleted file mode 100644
index 4dc61db..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/RecentMediaResponse.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.RecentMediaResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "data": {
- "type": "array",
- "items": {
- "$ref": "../pojo/Media.json"
- }
- },
- "meta": {
- "javaType": "org.apache.streams.instagram.api.Meta"
- },
- "pagination": {
- "javaType": "org.apache.streams.instagram.api.Pagination"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SearchMediaRequest.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SearchMediaRequest.json
deleted file mode 100644
index da1b790..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SearchMediaRequest.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.SearchMediaRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "distance": {
- "description": "Default is 1km (distance=1000), max distance is 5km.",
- "type": "integer"
- },
- "lat": {
- "description": "Latitude of the center search coordinate. If used, lng is required.",
- "type": "integer"
- },
- "lng": {
- "description": "Longitude of the center search coordinate. If used, lat is required.",
- "type": "integer"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SearchMediaResponse.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SearchMediaResponse.json
deleted file mode 100644
index a95e3c2..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SearchMediaResponse.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.SearchMediaResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "data": {
- "type": "array",
- "items": {
- "$ref": "../pojo/Media.json"
- }
- },
- "meta": {
- "javaType": "org.apache.streams.instagram.api.Meta"
- },
- "pagination": {
- "javaType": "org.apache.streams.instagram.api.Pagination"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SearchUsersRequest.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SearchUsersRequest.json
deleted file mode 100644
index 4ec3064..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SearchUsersRequest.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.SearchUsersRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "q": {
- "description": "query",
- "type": "string"
- },
- "count": {
- "description": "count",
- "type": "integer"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SearchUsersResponse.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SearchUsersResponse.json
deleted file mode 100644
index 28a5a56..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SearchUsersResponse.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.SearchUsersResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "data": {
- "type": "array",
- "items": {
- "$ref": "../pojo/UserInfo.json"
- }
- },
- "meta": {
- "javaType": "org.apache.streams.instagram.api.Meta"
- },
- "pagination": {
- "javaType": "org.apache.streams.instagram.api.Pagination"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SelfLikedMediaRequest.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SelfLikedMediaRequest.json
deleted file mode 100644
index a56196e..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SelfLikedMediaRequest.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.SelfLikedMediaRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "max_like_id": {
- "description": "max_like_id",
- "type": "integer"
- },
- "count": {
- "description": "count",
- "type": "integer"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SelfRecentMediaRequest.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SelfRecentMediaRequest.json
deleted file mode 100644
index 19d7a29..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/SelfRecentMediaRequest.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.SelfRecentMediaRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "max_id": {
- "description": "max_id",
- "type": "integer"
- },
- "min_id": {
- "description": "min_id",
- "type": "integer"
- },
- "count": {
- "description": "count",
- "type": "integer"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/UserInfoResponse.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/UserInfoResponse.json
deleted file mode 100644
index cc38cfd..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/UserInfoResponse.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.UserInfoResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "data": {
- "$ref": "../pojo/UserInfo.json"
- },
- "meta": {
- "javaType": "org.apache.streams.instagram.api.Meta"
- },
- "pagination": {
- "javaType": "org.apache.streams.instagram.api.Pagination"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/UserRecentMediaRequest.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/UserRecentMediaRequest.json
deleted file mode 100644
index 5ade8bc..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/UserRecentMediaRequest.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.pojo.UserRecentMediaRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "extends": {"$ref":"SelfRecentMediaRequest.json"},
- "properties": {
- "user_id": {
- "description": "user_id",
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/UsersInfoResponse.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/UsersInfoResponse.json
deleted file mode 100644
index 95c468d..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/api/UsersInfoResponse.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.api.UsersInfoResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "data": {
- "type": "array",
- "items": {
- "$ref": "../pojo/UserInfo.json"
- }
- },
- "meta": {
- "javaType": "org.apache.streams.instagram.api.Meta"
- },
- "pagination": {
- "javaType": "org.apache.streams.instagram.api.Pagination"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/config/InstagramConfiguration.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/config/InstagramConfiguration.json
deleted file mode 100644
index e213445..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/config/InstagramConfiguration.json
+++ /dev/null
@@ -1,64 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.instagram.config.InstagramConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "protocol": {
- "type": "string",
- "description": "The protocol",
- "default": "https"
- },
- "host": {
- "type": "string",
- "description": "The host",
- "default": "api.instagram.com"
- },
- "port": {
- "type": "integer",
- "description": "The port",
- "default": 443
- },
- "version": {
- "type": "string",
- "description": "The version",
- "default": "v1"
- },
- "oauth": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.instagram.config.InstagramOAuthConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "clientId": {
- "type": "string"
- },
- "clientSecret": {
- "type": "string"
- },
- "accessToken": {
- "type": "string"
- }
- }
- },
- "threadsPerProvider": {
- "type": "integer",
- "description": "number of threads per provider",
- "default": 10
- },
- "retrySleepMs": {
- "type": "integer",
- "description": "ms to sleep when hitting a rate limit",
- "default": 100000
- },
- "retryMax": {
- "type": "integer",
- "description": "max times to retry",
- "default": 10
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/config/InstagramRecentMediaProviderConfiguration.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/config/InstagramRecentMediaProviderConfiguration.json
deleted file mode 100644
index 8c1dc07..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/config/InstagramRecentMediaProviderConfiguration.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.instagram.config.InstagramRecentMediaProviderConfiguration",
- "extends": {"$ref":"InstagramConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "info": {
- "type": "array",
- "description": "A list of user IDs, indicating the users whose posts should be delivered on the stream",
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/config/InstagramUserInfoProviderConfiguration.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/config/InstagramUserInfoProviderConfiguration.json
deleted file mode 100644
index 9906c34..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/config/InstagramUserInfoProviderConfiguration.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.instagram.config.InstagramUserInfoProviderConfiguration",
- "extends": {"$ref":"InstagramConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "info": {
- "type": "array",
- "description": "A list of user IDs, indicating the users whose posts should be delivered on the stream",
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/pojo/Comment.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/pojo/Comment.json
deleted file mode 100644
index 5e7d60e..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/pojo/Comment.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.pojo.Comment",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "id": {
- "type": "string"
- },
- "created_time": {
- "type": "string"
- },
- "text": {
- "type": "string"
- },
- "from": {
- "$ref": "UserInfo.json"
- }
- }
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/pojo/Media.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/pojo/Media.json
deleted file mode 100644
index 5256e3a..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/pojo/Media.json
+++ /dev/null
@@ -1,164 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.pojo.Media",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "caption": {
- "type": "object",
- "javaType": "org.apache.streams.instagram.pojo.Caption",
- "properties": {
- "created_time": {
- "type": "string"
- },
- "from": {
- "$ref": "UserInfo.json"
- },
- "id": {
- "type": "string"
- },
- "text": {
- "type": "string"
- }
- }
- },
- "comments": {
- "type": "object",
- "javaType": "org.apache.streams.instagram.pojo.Comments",
- "properties": {
- "count": {
- "type": "integer"
- },
- "data": {
- "type": "array",
- "items": {
- "$ref": "Comment.json"
- }
- }
- }
- },
- "created_time": {
- "type": "string"
- },
- "filter": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "images": {
- "type": "object",
- "properties": {
- "low_resolution": {
- "description": "this should be org.apache.streams.pojo.json.MediaLink, ../../../../../../../../../../streams-schemas/streams-schema-activitystreams/src/main/jsonschema/media_link.json",
- "type": "object",
- "javaType": "org.apache.streams.pojo.json.ImageParent"
- },
- "standard_resolution": {
- "description": "this should be org.apache.streams.pojo.json.MediaLink, ../../../../../../../../../../streams-schemas/streams-schema-activitystreams/src/main/jsonschema/media_link.json",
- "type": "object",
- "javaType": "org.apache.streams.pojo.json.ImageParent"
- },
- "thumbnail": {
- "description": "this should be org.apache.streams.pojo.json.MediaLink, ../../../../../../../../../../streams-schemas/streams-schema-activitystreams/src/main/jsonschema/media_link.json",
- "type": "object",
- "javaType": "org.apache.streams.pojo.json.ImageParent"
- }
- }
- },
- "likes": {
- "type": "object",
- "javaType": "org.apache.streams.instagram.pojo.LikeCounts",
- "properties": {
- "count": {
- "type": "integer"
- }
- }
- },
- "link": {
- "type": "string"
- },
- "location": {
- "type": "object",
- "properties": {
- "id": {
- "type": "string"
- },
- "latitude": {
- "type": "number"
- },
- "longitude": {
- "type": "number"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "tags": {
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "type": {
- "type": "string",
- "enum": [
- "image",
- "video"
- ]
- },
- "user": {
- "$ref": "UserInfo.json"
- },
- "videos": {
- "type": "object",
- "properties": {
- "low_resolution": {
- "description": "this should be org.apache.streams.pojo.json.MediaLink, ../../../../../../../../../../streams-schemas/streams-schema-activitystreams/src/main/jsonschema/media_link.json",
- "type": "object",
- "javaType": "org.apache.streams.pojo.json.ImageParent"
- },
- "standard_resolution": {
- "description": "this should be org.apache.streams.pojo.json.MediaLink, ../../../../../../../../../../streams-schemas/streams-schema-activitystreams/src/main/jsonschema/media_link.json",
- "type": "object",
- "javaType": "org.apache.streams.pojo.json.ImageParent"
- },
- "thumbnail": {
- "description": "this should be org.apache.streams.pojo.json.MediaLink, ../../../../../../../../../../streams-schemas/streams-schema-activitystreams/src/main/jsonschema/media_link.json",
- "type": "object",
- "javaType": "org.apache.streams.pojo.json.ImageParent"
- }
- }
- },
- "users_in_photo": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "user": {
- "$ref": "UserInfo.json"
- },
- "position": {
- "type": "object",
- "properties": {
- "x": {
- "type": "number"
- },
- "y": {
- "type": "number"
- }
- }
- }
- }
- }
- }
-
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/pojo/UserInfo.json b/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/pojo/UserInfo.json
deleted file mode 100644
index b8af857..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/jsonschema/org/apache/streams/instagram/pojo/UserInfo.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.instagram.pojo.UserInfo",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "properties": {
- "id": {
- "type": "string"
- },
- "username": {
- "type": "string"
- },
- "full_name": {
- "type": "string"
- },
- "profile_picture": {
- "type": "string"
- },
- "bio": {
- "type": "string"
- },
- "website": {
- "type": "string"
- },
- "counts": {
- "type": "object",
- "javaType": "org.apache.streams.instagram.pojo.UserInfoCounts",
- "properties": {
- "media": {
- "type": "integer"
- },
- "follows": {
- "type": "integer"
- },
- "followed_by": {
- "type": "integer"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/main/resources/components.dot b/streams-contrib/streams-provider-instagram/src/main/resources/components.dot
deleted file mode 100644
index c5c0951..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/resources/components.dot
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- subgraph cluster_generators {
- label="generators";
- style="dotted";
- generators_instagram [label="https://api.instagram.com"]
- }
-
- subgraph cluster_providers {
- label="providers";
- providers_instagram_userinfo [label="InstagramUserInfoCollector"]
- providers_instagram_media [label="InstagramRecentMediaCollector"]
- }
-
- subgraph cluster_processors {
- label="processors";
- processors_instagram_activity [label="InstagramTypeConverter"]
- processors_instagram_activityobject [label="InstagramTypeConverter"]
- }
-
- persisters [label="persisters", shape="cylinder"];
-
- generators_instagram -> providers_instagram_userinfo
- generators_instagram -> providers_instagram_media
- providers_instagram_userinfo -> processors_instagram_activityobject [label="o.a.s.i.UserInfoData"]
- providers_instagram_media -> processors_instagram_activity [label="o.a.s.i.MediaFeedData"]
-
- processors_instagram_activityobject -> persisters [label="o.a.s.p.j.Page"]
- processors_instagram_activity -> persisters [label="o.a.s.p.j.Photo\no.a.s.p.j.Video\no.a.s.p.j.Comment"]
-
-}
diff --git a/streams-contrib/streams-provider-instagram/src/main/resources/reference.conf b/streams-contrib/streams-provider-instagram/src/main/resources/reference.conf
deleted file mode 100644
index ceca328..0000000
--- a/streams-contrib/streams-provider-instagram/src/main/resources/reference.conf
+++ /dev/null
@@ -1,11 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-instagram {
- protocol = "https"
- host = "api.instagram.com"
- port = 443
- version = "v1"
- threadsPerProvider = 10
- retrySleepMs = 100000
- retryMax = 10
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/site/markdown/index.md b/streams-contrib/streams-provider-instagram/src/site/markdown/index.md
deleted file mode 100644
index 5cb60b0..0000000
--- a/streams-contrib/streams-provider-instagram/src/site/markdown/index.md
+++ /dev/null
@@ -1,50 +0,0 @@
-streams-provider-instagram
-==========================
-
-Module connects to instagram API, collects events, converts to activity, and passes each activity downstream.
-
-## Data Types
-
-This module relies on classes from com.sachinhandiekar:jInstagram
-
-## Configuration
-
-| Schema |
-|--------|
-| [InstagramConfiguration.json](com/instagram/InstagramConfiguration.json "InstagramConfiguration.json") [InstagramConfiguration.html](apidocs/com/instagram/InstagramConfiguration.html "javadoc") |
-| [InstagramUserInformationConfiguration.json](com/instagram/InstagramUserInformationConfiguration.json "InstagramUserInformationConfiguration.json") [InstagramUserInformationConfiguration.html](apidocs/com/instagram/InstagramUserInformationConfiguration.html "javadoc") |
-
-## Components
-
-
-
-Test:
------
-
-Create a local file `instagram.conf` with valid instagram credentials
-
- instagram {
- clientId = "b389fcbeca2a40a89afa591a8468e4dc"
- usersInfo = {
- authorizedTokens = [
- "1646021441.b739937.58d5b84abce74241b640d8c1f7e91222"
- ]
- }
- }
-
-Build with integration testing enabled, using your credentials
-
- mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/instagram.conf"
-
-If you only have sandbox credentials, you'll have to edit src/test/resources/*IT.conf to pull ids that have been authorized on your sandbox.
-
-Resources:
-----------
-
-[https://elfsight.com/blog/2016/05/how-to-get-instagram-access-token/](https://elfsight.com/blog/2016/05/how-to-get-instagram-access-token/ "https://elfsight.com/blog/2016/05/how-to-get-instagram-access-token/")
-
-[https://smashballoon.com/instagram-feed/find-instagram-user-id/](https://smashballoon.com/instagram-feed/find-instagram-user-id/ "https://smashballoon.com/instagram-feed/find-instagram-user-id/")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/api/InstagramOAuthRequestSignerTest.java b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/api/InstagramOAuthRequestSignerTest.java
deleted file mode 100644
index 211dc25..0000000
--- a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/api/InstagramOAuthRequestSignerTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.test.api;
-
-import org.apache.streams.instagram.api.InstagramOAuthRequestSigner;
-import org.apache.streams.instagram.config.InstagramOAuthConfiguration;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Created by sblackmon on 3/26/17.
- */
-public class InstagramOAuthRequestSignerTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramOAuthRequestSignerTest.class);
-
- private static final InstagramOAuthConfiguration testOauthConfig = new InstagramOAuthConfiguration()
- .withClientSecret("6dc1787668c64c939929c17683d7cb74")
- .withAccessToken("fb2e77d.47a0479900504cb3ab4a1f626d174d2d");
-
- private static final InstagramOAuthRequestSigner signer =
- new InstagramOAuthRequestSigner(testOauthConfig);
-
- @Test
- public void testExample1() throws Exception {
- String endpoint = "/users/self";
- Map<String,String> testParamMap = new HashMap<>();
- testParamMap.put("access_token", "fb2e77d.47a0479900504cb3ab4a1f626d174d2d");
- String signature_parameter_string = signer.generateSignatureBaseString(endpoint, testParamMap);
- String expected_parameter_string = endpoint + "|access_token=fb2e77d.47a0479900504cb3ab4a1f626d174d2d";
- assertEquals(expected_parameter_string, signature_parameter_string);
- String signature = signer.computeSignature(signature_parameter_string, testOauthConfig.getClientSecret());
- String expected_signature = "cbf5a1f41db44412506cb6563a3218b50f45a710c7a8a65a3e9b18315bb338bf";
- assertEquals(expected_signature, signature);
- }
-
- @Test
- public void testExample2() throws Exception {
- String endpoint = "/media/657988443280050001_25025320";
- Map<String,String> testParamMap = new HashMap<>();
- testParamMap.put("access_token", "fb2e77d.47a0479900504cb3ab4a1f626d174d2d");
- testParamMap.put("count", "10");
- String signature_parameter_string = signer.generateSignatureBaseString(endpoint, testParamMap);
- String expected_parameter_string = endpoint + "|access_token=fb2e77d.47a0479900504cb3ab4a1f626d174d2d|count=10";
- assertEquals(expected_parameter_string, signature_parameter_string);
- String signature = signer.computeSignature(signature_parameter_string, testOauthConfig.getClientSecret());
- String expected_signature = "260634b241a6cfef5e4644c205fb30246ff637591142781b86e2075faf1b163a";
- assertEquals(expected_signature, signature);
- }
-
-}
diff --git a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramMediaFeedDataConverterIT.java b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramMediaFeedDataConverterIT.java
deleted file mode 100644
index 9c4a7da..0000000
--- a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramMediaFeedDataConverterIT.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.test.data;
-
-import org.apache.streams.data.ActivityConverter;
-import org.apache.streams.instagram.pojo.Media;
-import org.apache.streams.instagram.serializer.InstagramMediaFeedDataConverter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.PrintStream;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.junit.Assert.assertThat;
-
-/**
- * Tests conversion of instagram inputs to Activity.
- */
-public class InstagramMediaFeedDataConverterIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramMediaFeedDataConverterIT.class);
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Test(dependsOnGroups = "providers")
- public void InstagramMediaFeedDataConverterIT() throws Exception {
- InputStream is = InstagramMediaFeedDataConverterIT.class.getResourceAsStream("/InstagramRecentMediaProviderIT.stdout.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- PrintStream outStream = new PrintStream(
- new BufferedOutputStream(
- new FileOutputStream("target/test-classes/InstagramMediaFeedDataConverterIT.txt")));
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
-
- Media mediaFeedData = mapper.readValue(line, Media.class);
-
- ActivityConverter<Media> converter = new InstagramMediaFeedDataConverter();
-
- Activity activity = converter.toActivityList(mediaFeedData).get(0);
-
- LOGGER.info("activity: {}", activity.toString());
-
- assertThat(activity, is(not(nullValue())));
-
- assertThat(activity.getId(), is(not(nullValue())));
- assertThat(activity.getActor(), is(not(nullValue())));
- assertThat(activity.getActor().getId(), is(not(nullValue())));
- assertThat(activity.getVerb(), is(not(nullValue())));
- assertThat(activity.getProvider(), is(not(nullValue())));
-
- outStream.println(mapper.writeValueAsString(activity));
-
- }
-
- }
- outStream.flush();
-
- } catch ( Exception ex ) {
- LOGGER.error("Exception: ", ex);
- outStream.flush();
- Assert.fail();
- }
- }
-
-}
diff --git a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramUserInfoDataConverterIT.java b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramUserInfoDataConverterIT.java
deleted file mode 100644
index 91d4647..0000000
--- a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramUserInfoDataConverterIT.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.test.data;
-
-import org.apache.streams.data.ActivityObjectConverter;
-import org.apache.streams.instagram.pojo.UserInfo;
-import org.apache.streams.instagram.serializer.InstagramUserInfoDataConverter;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.gson.Gson;
-import org.apache.commons.lang.StringUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.PrintStream;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.junit.Assert.assertThat;
-
-/**
- * Tests conversion of instagram inputs to Activity.
- */
-public class InstagramUserInfoDataConverterIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoDataConverterIT.class);
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Test(dependsOnGroups = "providers")
- public void InstagramUserInfoDataConverterIT() throws Exception {
- InputStream is = InstagramUserInfoDataConverterIT.class.getResourceAsStream("/InstagramUserInfoProviderIT.stdout.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- PrintStream outStream = new PrintStream(
- new BufferedOutputStream(
- new FileOutputStream("target/test-classes/InstagramUserInfoDataConverterIT.txt")));
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
-
- LOGGER.info("raw: {}", line);
-
- UserInfo userInfoData = mapper.readValue(line, UserInfo.class);
-
- ActivityObjectConverter<UserInfo> converter = new InstagramUserInfoDataConverter();
-
- ActivityObject activityObject = converter.toActivityObject(userInfoData);
-
- LOGGER.info("activityObject: {}", activityObject.toString());
-
- assertThat(activityObject, is(not(nullValue())));
-
- assertThat(activityObject.getId(), is(not(nullValue())));
- assertThat(activityObject.getImage(), is(not(nullValue())));
- assertThat(activityObject.getDisplayName(), is(not(nullValue())));
- assertThat(activityObject.getSummary(), is(not(nullValue())));
-
- Map<String, Object> extensions = (Map<String, Object>)activityObject.getAdditionalProperties().get("extensions");
- assertThat(extensions, is(not(nullValue())));
- assertThat(extensions.get("following"), is(not(nullValue())));
- assertThat(extensions.get("followers"), is(not(nullValue())));
- assertThat(extensions.get("screenName"), is(not(nullValue())));
- assertThat(extensions.get("posts"), is(not(nullValue())));
-
- assertThat(activityObject.getAdditionalProperties().get("handle"), is(not(nullValue())));
- assertThat(activityObject.getId(), is(not(nullValue())));
- assertThat(activityObject.getUrl(), is(not(nullValue())));
-
- assertThat(activityObject.getAdditionalProperties().get("provider"), is(not(nullValue())));
-
- outStream.println(mapper.writeValueAsString(activityObject));
-
- }
- }
- outStream.flush();
-
- } catch ( Exception ex ) {
- LOGGER.error("Exception: ", ex);
- outStream.flush();
- Assert.fail();
- }
- }
-}
diff --git a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramRecentMediaProviderIT.java b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramRecentMediaProviderIT.java
deleted file mode 100644
index f5c55ed..0000000
--- a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramRecentMediaProviderIT.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.test.providers;
-
-import org.apache.streams.instagram.provider.recentmedia.InstagramRecentMediaProvider;
-
-import org.hamcrest.Matchers;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-
-/**
- * Integration Test for InstagramRecentMediaProvider.
- */
-public class InstagramRecentMediaProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramRecentMediaProviderIT.class);
-
- @Test(groups = "providers")
- public void testInstagramRecentMediaProvider() throws Exception {
-
- String configfile = "./target/test-classes/InstagramRecentMediaProviderIT.conf";
- String outfile = "./target/test-classes/InstagramRecentMediaProviderIT.stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- InstagramRecentMediaProvider.main(args);
- } catch ( Exception ex ) {
- LOGGER.error("Test Exception!", ex);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- File out = new File(outfile);
- Assert.assertTrue(out.exists());
- Assert.assertTrue(out.canRead());
- Assert.assertTrue(out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- assertThat(outCounter.getLineNumber(), Matchers.greaterThanOrEqualTo(1));
-
- }
-}
diff --git a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramUserInfoProviderIT.java b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramUserInfoProviderIT.java
deleted file mode 100644
index 8e00084..0000000
--- a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramUserInfoProviderIT.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.instagram.test.providers;
-
-import org.apache.streams.instagram.provider.userinfo.InstagramUserInfoProvider;
-
-import org.hamcrest.Matchers;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-
-public class InstagramUserInfoProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoProviderIT.class);
-
- @Test(groups = "providers")
- public void testInstagramUserInfoProvider() throws Exception {
-
- String configfile = "./target/test-classes/InstagramUserInfoProviderIT.conf";
- String outfile = "./target/test-classes/InstagramUserInfoProviderIT.stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- InstagramUserInfoProvider.main(args);
- } catch ( Exception ex ) {
- LOGGER.error("Test Exception!", ex);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- File out = new File(outfile);
- Assert.assertTrue(out.exists());
- Assert.assertTrue(out.canRead());
- Assert.assertTrue(out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- assertThat(outCounter.getLineNumber(), Matchers.greaterThanOrEqualTo(1));
-
- }
-}
diff --git a/streams-contrib/streams-provider-instagram/src/test/resources/InstagramRecentMediaProviderIT.conf b/streams-contrib/streams-provider-instagram/src/test/resources/InstagramRecentMediaProviderIT.conf
deleted file mode 100644
index 3bfb7c0..0000000
--- a/streams-contrib/streams-provider-instagram/src/test/resources/InstagramRecentMediaProviderIT.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-instagram {
- info = [
- #blackmonsteve
- "1646021441"
- ]
- retryMax = 1
- retrySleepMs = 1000
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-instagram/src/test/resources/InstagramUserInfoProviderIT.conf b/streams-contrib/streams-provider-instagram/src/test/resources/InstagramUserInfoProviderIT.conf
deleted file mode 100644
index 3bfb7c0..0000000
--- a/streams-contrib/streams-provider-instagram/src/test/resources/InstagramUserInfoProviderIT.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-instagram {
- info = [
- #blackmonsteve
- "1646021441"
- ]
- retryMax = 1
- retrySleepMs = 1000
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-moreover/README.md b/streams-contrib/streams-provider-moreover/README.md
deleted file mode 100644
index 765a1e4..0000000
--- a/streams-contrib/streams-provider-moreover/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-provider-moreover
-============================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-provider-moreover/pom.xml b/streams-contrib/streams-provider-moreover/pom.xml
deleted file mode 100644
index 761001a..0000000
--- a/streams-contrib/streams-provider-moreover/pom.xml
+++ /dev/null
@@ -1,190 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-provider-moreover</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Moreover Provider</description>
-
- <properties>
- <skipITs>true</skipITs>
- <testDataBaseURl>http://streams.peoplepattern.com.s3.amazonaws.com/test-data/</testDataBaseURl>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>com.moreover</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.jvnet.jaxb2.maven2</groupId>
- <artifactId>maven-jaxb2-plugin</artifactId>
- <configuration>
- <schemaDirectory>src/main/xmlschema/com/moreover/</schemaDirectory>
- <generateDirectory>target/generated-sources/jaxb2</generateDirectory>
- <verbose>true</verbose>
- <debug>true</debug>
- <encoding>${project.build.sourceEncoding}</encoding>
- <forceRegenerate>true</forceRegenerate>
- <removeOldOutput>false</removeOldOutput>
- <generatePackage>com.moreover.api</generatePackage>
- <plugins>
- <plugin>
- <groupId>org.jvnet.jaxb2_commons</groupId>
- <artifactId>jaxb2-basics</artifactId>
- <version>${jaxb2-basics.version}</version>
- </plugin>
- </plugins>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- <source>target/generated-sources/jaxb2</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <version>1.2.1</version>
- <executions>
- <execution>
- <id>download-it-data</id>
- <phase>pre-integration-test</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
- <url>${testDataBaseURl}/${project.artifactId}.zip</url>
- <unpack>true</unpack>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- <!--<md5>df65b5642f33676313ebe4d5b69a3fff</md5>-->
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <configuration>
- <skipTests>${skipITs}</skipTests>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverClient.java b/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverClient.java
deleted file mode 100644
index d9774ed..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverClient.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.moreover;
-
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.StringWriter;
-import java.math.BigInteger;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.nio.charset.Charset;
-import java.util.Date;
-
-/**
- * MoreoverClient is a Client for Moreover.
- */
-public class MoreoverClient {
- private static final Logger logger = LoggerFactory.getLogger(MoreoverClient.class);
-
- private static final String BASE_URL = "http://metabase.moreover.com/api/v10/articles?key=%s&limit=%s&sequence_id=%s";
- private final String id;
- private String apiKey;
- private BigInteger lastSequenceId = BigInteger.ZERO;
- //testing purpose only
- public long pullTime;
- private boolean debug;
-
- /**
- * MoreoverClient constructor.
- * @param id id
- * @param apiKey apiKey
- * @param sequence sequence
- */
- public MoreoverClient(String id, String apiKey, String sequence) {
- logger.info("Constructed new client for id:{} key:{} sequence:{}", id, apiKey, sequence);
- this.id = id;
- this.apiKey = apiKey;
- this.lastSequenceId = new BigInteger(sequence);
- }
-
- /**
- * get limit ArticlesAfter sequenceId.
- * @param sequenceId sequenceId
- * @param limit limit
- * @return MoreoverResult
- * @throws IOException IOException
- */
- public MoreoverResult getArticlesAfter(String sequenceId, int limit) throws IOException {
- String urlString = String.format(BASE_URL, this.apiKey, limit, sequenceId);
- logger.debug("Making call to {}", urlString);
- long start = System.nanoTime();
- MoreoverResult result = new MoreoverResult(id, getArticles(new URL(urlString)), start, System.nanoTime());
- if (!result.getMaxSequencedId().equals(BigInteger.ZERO)) {
- this.lastSequenceId = result.getMaxSequencedId();
- logger.debug("Maximum sequence from last call {}", this.lastSequenceId);
- } else {
- logger.debug("No maximum sequence returned in last call {}", this.lastSequenceId);
- }
- return result;
- }
-
- public MoreoverResult getNextBatch() throws IOException {
- logger.debug("Getting next results for {} {} {}", this.id, this.apiKey, this.lastSequenceId);
- return getArticlesAfter(this.lastSequenceId.toString(), 500);
- }
-
- private String getArticles2(URL url) throws IOException {
- HttpURLConnection cn = (HttpURLConnection) url.openConnection();
- cn.setRequestMethod("GET");
- cn.addRequestProperty("Content-Type", "text/xml;charset=UTF-8");
- cn.setDoInput(true);
- cn.setDoOutput(false);
- BufferedReader reader = new BufferedReader(new InputStreamReader(cn.getInputStream(), Charset.forName("UTF-8")));
- String line = null;
- StringBuilder builder = new StringBuilder();
- String result = new String("".getBytes(Charset.forName("UTF-8")), Charset.forName("UTF-8"));
- while ((line = reader.readLine()) != null) {
- result += line;
- }
- pullTime = new Date().getTime();
- return result;
- }
-
- private String getArticles(URL url) throws IOException {
- HttpURLConnection cn = (HttpURLConnection) url.openConnection();
- cn.setRequestMethod("GET");
- cn.addRequestProperty("Content-Type", "text/xml;charset=UTF-8");
- cn.setDoInput(true);
- cn.setDoOutput(false);
- StringWriter writer = new StringWriter();
- IOUtils.copy(new InputStreamReader(cn.getInputStream(), Charset.forName("UTF-8")), writer);
- writer.flush();
- pullTime = new Date().getTime();
-
- // added after seeing java.net.SocketException: Too many open files
- cn.disconnect();
-
- return writer.toString();
- }
-}
diff --git a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverJsonActivitySerializer.java b/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverJsonActivitySerializer.java
deleted file mode 100644
index 91d487e..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverJsonActivitySerializer.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.moreover;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.AnnotationIntrospector;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector;
-import com.moreover.api.Article;
-import org.apache.commons.lang.NotImplementedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Deserializes Moreover JSON format into Activities.
- */
-public class MoreoverJsonActivitySerializer implements ActivitySerializer<String> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MoreoverJsonActivitySerializer.class);
-
- public MoreoverJsonActivitySerializer() {
- }
-
- @Override
- public String serializationFormat() {
- return "application/json+vnd.moreover.com.v1";
- }
-
- @Override
- public String serialize(Activity deserialized) {
- throw new UnsupportedOperationException("Cannot currently serialize to Moreover JSON");
- }
-
- @Override
- public Activity deserialize(String serialized) {
- serialized = serialized.replaceAll("\\[[ ]*\\]", "null");
-
- LOGGER.debug(serialized);
-
- ObjectMapper mapper = new ObjectMapper();
- AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(mapper.getTypeFactory());
- mapper.setAnnotationIntrospector(introspector);
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, Boolean.FALSE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
- mapper.configure(DeserializationFeature.WRAP_EXCEPTIONS, Boolean.TRUE);
-
- Article article;
- try {
- ObjectNode node = (ObjectNode)mapper.readTree(serialized);
- node.remove("tags");
- node.remove("locations");
- node.remove("companies");
- node.remove("topics");
- node.remove("media");
- node.remove("outboundUrls");
- ObjectNode jsonNodes = (ObjectNode) node.get("source").get("feed");
- jsonNodes.remove("editorialTopics");
- jsonNodes.remove("tags");
- jsonNodes.remove("autoTopics");
- article = mapper.convertValue(node, Article.class);
- } catch (IOException ex) {
- throw new IllegalArgumentException("Unable to deserialize", ex);
- }
- return MoreoverUtils.convert(article);
- }
-
- @Override
- public List<Activity> deserializeAll(List<String> serializedList) {
- throw new NotImplementedException("Not currently implemented");
- }
-
-
-}
diff --git a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverProvider.java b/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverProvider.java
deleted file mode 100644
index a295898..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverProvider.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.moreover;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Streams Provider for the Moreover Metabase API.
- */
-public class MoreoverProvider implements StreamsProvider {
-
- public static final String STREAMS_ID = "MoreoverProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MoreoverProvider.class);
-
- protected volatile Queue<StreamsDatum> providerQueue = new ConcurrentLinkedQueue<>();
-
- private List<MoreoverKeyData> keys;
-
- private MoreoverConfiguration config;
-
- private ExecutorService executor;
-
- /**
- * MoreoverProvider constructor.
- * @param moreoverConfiguration MoreoverConfiguration
- */
- public MoreoverProvider(MoreoverConfiguration moreoverConfiguration) {
- this.config = moreoverConfiguration;
- this.keys = new ArrayList<>();
- for ( MoreoverKeyData apiKey : config.getApiKeys()) {
- this.keys.add(apiKey);
- }
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
-
- for (MoreoverKeyData key : keys) {
- MoreoverProviderTask task = new MoreoverProviderTask(key.getId(), key.getKey(), this.providerQueue, key.getStartingSequence());
- executor.submit(new Thread(task));
- LOGGER.info("Started producer for {}", key.getKey());
- }
-
- }
-
- @Override
- public synchronized StreamsResultSet readCurrent() {
-
- LOGGER.debug("readCurrent: {}", providerQueue.size());
-
- Collection<StreamsDatum> currentIterator = new ArrayList<>();
- Iterators.addAll(currentIterator, providerQueue.iterator());
-
- StreamsResultSet current = new StreamsResultSet(new ConcurrentLinkedQueue<>(currentIterator));
-
- providerQueue.clear();
-
- return current;
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return !executor.isShutdown() && !executor.isTerminated();
- }
-
- @Override
- public void prepare(Object configurationObject) {
- LOGGER.debug("Prepare");
- executor = Executors.newSingleThreadExecutor();
- }
-
- @Override
- public void cleanUp() {
-
- }
-
- /**
- * To use from command line:
- *
- * <p/>
- * Supply configuration similar to src/test/resources/rss.conf
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.moreover.MoreoverProvider -Dexec.args="rss.conf articles.json"
- *
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- MoreoverConfiguration config = new ComponentConfigurator<>(MoreoverConfiguration.class).detectConfiguration(typesafe, "rss");
- MoreoverProvider provider = new MoreoverProvider(config);
-
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- try {
- json = mapper.writeValueAsString(datum.getDocument());
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning() );
- provider.cleanUp();
- outStream.flush();
- }
-}
diff --git a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverProviderTask.java b/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverProviderTask.java
deleted file mode 100644
index 595fb4f..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverProviderTask.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.moreover;
-
-import org.apache.streams.core.StreamsDatum;
-
-import com.google.common.collect.ImmutableSet;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Queue;
-
-/**
- * Task that pulls from the Morever API on behalf of MoreoverProvider.
- */
-public class MoreoverProviderTask implements Runnable {
-
- public static final int LATENCY = 10;
- public static final int REQUIRED_LATENCY = LATENCY * 1000;
- private static Logger logger = LoggerFactory.getLogger(MoreoverProviderTask.class);
-
- private String lastSequence;
- private final String apiKey;
- private final Queue<StreamsDatum> results;
- private final MoreoverClient moClient;
- private boolean started = false;
-
- /**
- * MoreoverProviderTask constructor.
- * @param apiId apiId
- * @param apiKey apiKey
- * @param results results
- * @param lastSequence lastSequence
- */
- public MoreoverProviderTask(String apiId, String apiKey, Queue<StreamsDatum> results, String lastSequence) {
- //logger.info("Constructed new task {} for {} {} {}", UUID.randomUUID().toString(), apiId, apiKey, lastSequence);
- this.apiKey = apiKey;
- this.results = results;
- this.lastSequence = lastSequence;
- this.moClient = new MoreoverClient(apiId, this.apiKey, this.lastSequence);
- initializeClient(moClient);
- }
-
- @Override
- public void run() {
- while (true) {
- try {
- ensureTime(moClient);
- MoreoverResult result = moClient.getArticlesAfter(lastSequence, 500);
- started = true;
- lastSequence = result.process().toString();
- for (StreamsDatum entry : ImmutableSet.copyOf(result.iterator())) {
- results.offer(entry);
- }
- logger.info("ApiKey={}\tlastSequenceid={}", this.apiKey, lastSequence);
-
- } catch (Exception ex) {
- logger.error("Exception while polling moreover", ex);
- }
- }
- }
-
- private void ensureTime(MoreoverClient moClient) {
- try {
- long gap = System.currentTimeMillis() - moClient.pullTime;
- if (gap < REQUIRED_LATENCY) {
- Thread.sleep(REQUIRED_LATENCY - gap);
- }
- } catch (Exception ex) {
- logger.warn("Error sleeping for latency");
- }
- }
-
- private void initializeClient(MoreoverClient moClient) {
- try {
- moClient.getArticlesAfter(this.lastSequence, 2);
- } catch (Exception ex) {
- logger.error("Failed to start stream, {}", this.apiKey);
- logger.error("Exception : ", ex);
- throw new IllegalStateException("Unable to initialize stream", ex);
- }
- }
-}
diff --git a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverResult.java b/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverResult.java
deleted file mode 100644
index c3a78bb..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverResult.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.moreover;
-
-import org.apache.streams.core.StreamsDatum;
-
-import com.fasterxml.aalto.stax.InputFactoryImpl;
-import com.fasterxml.aalto.stax.OutputFactoryImpl;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule;
-import com.fasterxml.jackson.dataformat.xml.XmlFactory;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import com.moreover.api.Article;
-import com.moreover.api.ArticlesResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-public class MoreoverResult implements Iterable<StreamsDatum> {
-
- private static final Logger logger = LoggerFactory.getLogger(MoreoverResult.class);
-
- private XmlMapper xmlMapper;
-
- private String xmlString;
- private String jsonString;
- private ArticlesResponse resultObject;
- private ArticlesResponse.Articles articles;
- private List<Article> articleArray;
- private long start;
- private long end;
- private String clientId;
- private BigInteger maxSequencedId = BigInteger.ZERO;
-
- protected ArticlesResponse response;
- protected List<StreamsDatum> list = new ArrayList<>();
-
- protected MoreoverResult(String clientId, String xmlString, long start, long end) {
- this.xmlString = xmlString;
- this.clientId = clientId;
- this.start = start;
- this.end = end;
- XmlFactory xmlFactory = new XmlFactory(new InputFactoryImpl(),
- new OutputFactoryImpl());
-
- JacksonXmlModule module = new JacksonXmlModule();
-
- module.setDefaultUseWrapper(false);
-
- xmlMapper = new XmlMapper(xmlFactory, module);
-
- xmlMapper
- .configure(
- DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY,
- Boolean.TRUE);
- xmlMapper
- .configure(
- DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT,
- Boolean.TRUE);
- xmlMapper
- .configure(
- DeserializationFeature.USE_JAVA_ARRAY_FOR_JSON_ARRAY,
- Boolean.TRUE);
- xmlMapper.configure(
- DeserializationFeature.READ_ENUMS_USING_TO_STRING,
- Boolean.TRUE);
- xmlMapper.configure(
- DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,
- Boolean.FALSE);
-
- ObjectMapper mapper = new ObjectMapper();
-
- mapper
- .configure(
- DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY,
- Boolean.TRUE);
- mapper.configure(
- DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT,
- Boolean.TRUE);
- mapper
- .configure(
- DeserializationFeature.USE_JAVA_ARRAY_FOR_JSON_ARRAY,
- Boolean.TRUE);
- mapper.configure(
- DeserializationFeature.READ_ENUMS_USING_TO_STRING,
- Boolean.TRUE);
-
- }
-
- public String getClientId() {
- return clientId;
- }
-
- public long getStart() {
- return start;
- }
-
- public long getEnd() {
- return end;
- }
-
- /**
- * Process batch and
- * @return max sequenceId.
- */
- public BigInteger process() {
-
- try {
- this.resultObject = xmlMapper.readValue(xmlString, ArticlesResponse.class);
- } catch (JsonMappingException ex) {
- // theory is this may not be fatal
- this.resultObject = (ArticlesResponse) ex.getPath().get(0).getFrom();
- } catch (Exception ex) {
- ex.printStackTrace();
- logger.warn("Unable to process document:");
- logger.warn(xmlString);
- }
-
- if ( this.resultObject.getStatus().equals("FAILURE")) {
- logger.warn(this.resultObject.getStatus());
- logger.warn(this.resultObject.getMessageCode());
- logger.warn(this.resultObject.getUserMessage());
- logger.warn(this.resultObject.getDeveloperMessage());
- } else {
- this.articles = resultObject.getArticles();
- this.articleArray = articles.getArticle();
-
- for (Article article : articleArray) {
- BigInteger sequenceid = new BigInteger(article.getSequenceId());
- list.add(new StreamsDatum(article, sequenceid));
- logger.trace("Prior max sequence Id {} current candidate {}", this.maxSequencedId, sequenceid);
- if (sequenceid.compareTo(this.maxSequencedId) > 0) {
- this.maxSequencedId = sequenceid;
- }
- }
- }
-
- return this.maxSequencedId;
- }
-
- public String getXmlString() {
- return this.xmlString;
- }
-
- public BigInteger getMaxSequencedId() {
- return this.maxSequencedId;
- }
-
- @Override
- public Iterator<StreamsDatum> iterator() {
- return list.iterator();
- }
-
- protected static class JsonStringIterator implements Iterator<Serializable> {
-
- private Iterator<Serializable> underlying;
-
- protected JsonStringIterator(Iterator<Serializable> underlying) {
- this.underlying = underlying;
- }
-
- @Override
- public boolean hasNext() {
- return underlying.hasNext();
- }
-
- @Override
- public String next() {
- return underlying.next().toString();
- }
-
- @Override
- public void remove() {
- underlying.remove();
- }
- }
-}
diff --git a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverUtils.java b/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverUtils.java
deleted file mode 100644
index f9a3595..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverUtils.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.moreover;
-
-import org.apache.streams.data.util.ActivityUtil;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Provider;
-
-import com.moreover.api.Article;
-import com.moreover.api.Author;
-import com.moreover.api.AuthorPublishingPlatform;
-import com.moreover.api.Feed;
-import com.moreover.api.Source;
-import org.joda.time.DateTime;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.streams.data.util.ActivityUtil.LANGUAGE_EXTENSION;
-import static org.apache.streams.data.util.ActivityUtil.LOCATION_EXTENSION;
-import static org.apache.streams.data.util.ActivityUtil.LOCATION_EXTENSION_COUNTRY;
-import static org.apache.streams.data.util.ActivityUtil.getObjectId;
-
-/**
- * Provides utilities for Moreover data.
- */
-public class MoreoverUtils {
-
- private MoreoverUtils() {
- }
-
- public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'";
-
- /**
- * convert article into Activity.
- * @param article article
- * @return Activity
- */
- public static Activity convert(Article article) {
- Activity activity = new Activity();
- Source source = article.getSource();
- activity.setActor(convert(article.getAuthor(), source.getName()));
- activity.setProvider(convert(source));
- activity.setTarget(convertTarget(source));
- activity.setObject(convertObject(article));
- activity.setPublished(DateTime.parse(article.getPublishedDate()));
- activity.setContent(article.getContent());
- activity.setTitle(article.getTitle());
- activity.setVerb("posted");
- fixActivityId(activity);
- addLocationExtension(activity, source);
- addLanguageExtension(activity, article);
- activity.setLinks(convertLinks(article));
- return activity;
- }
-
- /**
- * convert Source to Provider.
- * @param source Source
- * @return Provider
- */
- public static Provider convert(Source source) {
- Provider provider = new Provider();
- Feed feed = source.getFeed();
- String display = getProviderId(feed);
- provider.setId(ActivityUtil.getProviderId(display.trim().toLowerCase().replace(" ", "_")));
- provider.setDisplayName(display);
- provider.setUrl(feed.getUrl());
- return provider;
- }
-
- /**
- * convert Author and platformName to Actor.
- * @param author Author
- * @param platformName platformName
- * @return $.actor
- */
- public static ActivityObject convert(Author author, String platformName) {
- ActivityObject actor = new ActivityObject();
- AuthorPublishingPlatform platform = author.getPublishingPlatform();
- String userId = platform.getUserId();
- if (userId != null) {
- actor.setId(ActivityUtil.getPersonId(getProviderId(platformName), userId));
- }
- actor.setDisplayName(author.getName());
- actor.setUrl(author.getHomeUrl());
- actor.setSummary(author.getDescription());
- actor.setAdditionalProperty("email", author.getEmail());
- return actor;
- }
-
- private static void fixActivityId(Activity activity) {
- if (activity.getId() != null && activity.getId().matches("\\{[a-z]*\\}")) {
- activity.setId(null);
- }
- }
-
- private static List convertLinks(Article article) {
- List<String> list = new LinkedList<>();
- Article.OutboundUrls outboundUrls = article.getOutboundUrls();
- if (outboundUrls != null) {
- for (String url : outboundUrls.getOutboundUrl()) {
- list.add(url);
- }
- }
- return list;
- }
-
- /**
- * convertTarget.
- * @param source source
- * @return ActivityObject $.target
- */
- public static ActivityObject convertTarget(Source source) {
- ActivityObject object = new ActivityObject();
- object.setUrl(source.getHomeUrl());
- object.setDisplayName(source.getName());
- return object;
- }
-
- /**
- * convertObject.
- * @param article article
- * @return ActivityObject $.object
- */
- public static ActivityObject convertObject(Article article) {
- ActivityObject object = new ActivityObject();
- object.setContent(article.getContent());
- object.setSummary(article.getTitle());
- object.setUrl(article.getOriginalUrl());
- object.setObjectType(article.getDataFormat());
- String type = article.getDataFormat().equals("text") ? "article" : article.getDataFormat();
- object.setId(getObjectId(getProviderId(article.getSource().getFeed()), type, article.getId()));
- object.setPublished(DateTime.parse(article.getPublishedDate()));
- return object;
- }
-
- /**
- * addLocationExtension.
- * @param activity Activity
- * @param source Source
- */
- public static void addLocationExtension(Activity activity, Source source) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- String country = source.getLocation().getCountryCode() == null
- ? source.getLocation().getCountry()
- : source.getLocation().getCountryCode();
- if (country != null) {
- Map<String, Object> location = new HashMap<>();
- location.put(LOCATION_EXTENSION_COUNTRY, country);
- extensions.put(LOCATION_EXTENSION, location);
- }
- }
-
- /**
- * addLanguageExtension.
- * @param activity Activity
- * @param article Article
- */
- public static void addLanguageExtension(Activity activity, Article article) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- String language = article.getLanguage();
- if (language != null) {
- extensions.put(LANGUAGE_EXTENSION, language);
- }
- }
-
- private static String getProviderId(Feed feed) {
- return getProviderId(feed.getPublishingPlatform() == null ? feed.getMediaType() : feed.getPublishingPlatform());
- }
-
- private static String getProviderId(String feed) {
- return feed.toLowerCase().replace(" ", "_").trim();
- }
-}
diff --git a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverXmlActivitySerializer.java b/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverXmlActivitySerializer.java
deleted file mode 100644
index fe4378c..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverXmlActivitySerializer.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.moreover;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.pojo.json.Activity;
-
-import com.moreover.api.Article;
-import com.moreover.api.ArticlesResponse;
-import com.moreover.api.ObjectFactory;
-import org.apache.commons.lang.SerializationException;
-
-import java.io.StringReader;
-import java.util.LinkedList;
-import java.util.List;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBElement;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Unmarshaller;
-
-/**
- * Deserializes the Moreover Article XML and converts it to an instance of {@link Activity}.
- */
-public class MoreoverXmlActivitySerializer implements ActivitySerializer<String> {
-
- //JAXBContext is threadsafe (supposedly)
- private final JAXBContext articleContext;
- private final JAXBContext articlesContext;
-
- public MoreoverXmlActivitySerializer() {
- articleContext = createContext(Article.class);
- articlesContext = createContext(ArticlesResponse.class);
- }
-
- @Override
- public String serializationFormat() {
- return "application/xml+vnd.moreover.com.v1";
- }
-
- @Override
- public String serialize(Activity deserialized) {
- throw new UnsupportedOperationException("Cannot currently serialize to Moreover");
- }
-
- @Override
- public Activity deserialize(String serialized) {
- Article article = deserializeMoreover(serialized);
- return MoreoverUtils.convert(article);
- }
-
- @Override
- public List<Activity> deserializeAll(List<String> serializedList) {
- List<Activity> activities = new LinkedList<Activity>();
- for (String item : serializedList) {
- ArticlesResponse response = deserializeMoreoverResponse(item);
- for (Article article : response.getArticles().getArticle()) {
- activities.add(MoreoverUtils.convert(article));
- }
- }
- return activities;
- }
-
- private Article deserializeMoreover(String serialized) {
- try {
- Unmarshaller unmarshaller = articleContext.createUnmarshaller();
- return (Article) unmarshaller.unmarshal(new StringReader(serialized));
- } catch (JAXBException ex) {
- throw new SerializationException("Unable to deserialize Moreover data", ex);
- }
- }
-
- private ArticlesResponse deserializeMoreoverResponse(String serialized) {
- try {
- Unmarshaller unmarshaller = articlesContext.createUnmarshaller();
- return ((JAXBElement<ArticlesResponse>) unmarshaller.unmarshal(new StringReader(serialized))).getValue();
- } catch (JAXBException ex) {
- throw new SerializationException("Unable to deserialize Moreover data", ex);
- }
- }
-
- private JAXBContext createContext(Class articleClass) {
- JAXBContext context;
- try {
- context = JAXBContext.newInstance(articleClass.getPackage().getName(), ObjectFactory.class.getClassLoader());
- } catch (JAXBException ex) {
- throw new IllegalStateException("Unable to create JAXB Context for Moreover data", ex);
- }
- return context;
- }
-}
diff --git a/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/Moreover.json b/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/Moreover.json
deleted file mode 100644
index ee60b3e..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/Moreover.json
+++ /dev/null
@@ -1,337 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.moreover.Moreover",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "tags": {
- "type": "string"
- },
- "publishingPlatform": {
- "properties": {
- "totalViews": {
- "type": "string"
- },
- "itemId": {
- "type": "string"
- }
- }
- },
- "loginStatus": {
- "type": "string"
- },
- "duplicateGroupId": {
- "type": "string"
- },
- "companies": {
- "properties": {
- "symbol": {
- "type": "string"
- },
- "primary": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "contentCount": {
- "type": "string"
- },
- "exchange": {
- "type": "string"
- },
- "titleCount": {
- "type": "string"
- },
- "isin": {
- "type": "string"
- }
- }
- },
- "copyright": {
- "type": "string"
- },
- "url": {
- "type": "string"
- },
- "content": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "author": {
- "properties": {
- "publishingPlatform": {
- "properties": {
- "userId": {
- "type": "string"
- },
- "totalViews": {
- "type": "string"
- }
- }
- },
- "dateLastActive": {
- "type": "string"
- },
- "description": {
- "type": "string"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "topics": {
- "properties": {
- "name": {
- "type": "string"
- },
- "group": {
- "type": "string"
- }
- }
- },
- "title": {
- "type": "string"
- },
- "source": {
- "properties": {
- "category": {
- "type": "string"
- },
- "location": {
- "properties": {
- "region": {
- "type": "string"
- },
- "subregion": {
- "type": "string"
- },
- "zipCode": {
- "type": "string"
- },
- "state": {
- "type": "string"
- },
- "countryCode": {
- "type": "string"
- },
- "zipArea": {
- "type": "string"
- },
- "country": {
- "type": "string"
- }
- }
- },
- "editorialRank": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "feed": {
- "properties": {
- "tags": {
- "type": "string"
- },
- "genre": {
- "type": "string"
- },
- "publishingPlatform": {
- "type": "string"
- },
- "inWhiteList": {
- "type": "string"
- },
- "imageUrl": {
- "type": "string"
- },
- "copyright": {
- "type": "string"
- },
- "mediaType": {
- "type": "string"
- },
- "id": {
- "type": "string"
- },
- "rank": {
- "properties": {
- "autoRankOrder": {
- "type": "string"
- },
- "inboundLinkCount": {
- "type": "string"
- },
- "autoRank": {
- "type": "string"
- }
- }
- },
- "description": {
- "type": "string"
- },
- "idFromPublisher": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "dataFormat": {
- "type": "string"
- },
- "generator": {
- "type": "string"
- },
- "autoTopics": {
- "type": "string"
- },
- "language": {
- "type": "string"
- },
- "editorialTopics": {
- "type": "string"
- }
- }
- },
- "homeUrl": {
- "type": "string"
- },
- "publisher": {
- "type": "string"
- }
- }
- },
- "locations": {
- "properties": {
- "region": {
- "type": "string"
- },
- "subregion": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "state": {
- "properties": {
- "name": {
- "type": "string"
- },
- "fipsCode": {
- "type": "string"
- },
- "confidence": {
- "type": "string"
- }
- }
- },
- "longitude": {
- "type": "string"
- },
- "latitude": {
- "type": "string"
- },
- "confidence": {
- "type": "string"
- },
- "type": {
- "type": "string"
- },
- "mentions": {
- "type": "string"
- },
- "country": {
- "properties": {
- "name": {
- "type": "string"
- },
- "fipsCode": {
- "type": "string"
- },
- "isoCode": {
- "type": "string"
- },
- "confidence": {
- "type": "string"
- }
- }
- }
- }
- },
- "commentsUrl": {
- "type": "string"
- },
- "dataFormat": {
- "type": "string"
- },
- "outboundUrls": {
- "type": "string"
- },
- "sequenceId": {
- "type": "string"
- },
- "publishedDate": {
- "type": "string"
- },
- "language": {
- "type": "string"
- },
- "adultLanguage": {
- "type": "string"
- },
- "harvestDate": {
- "type": "string"
- },
- "media": {
- "properties": {
- "duration": {
- "type": "string"
- },
- "audio": {
- "properties": {
- "url": {
- "type": "string"
- },
- "mimeType": {
- "type": "string"
- }
- }
- },
- "image": {
- "properties": {
- "url": {
- "type": "string"
- }
- }
- },
- "caption": {
- "type": "string"
- },
- "video": {
- "properties": {
- "url": {
- "type": "string"
- },
- "mimeType": {
- "type": "string"
- }
- }
- },
- "url": {
- "type": "string"
- },
- "mimeType": {
- "type": "string"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/MoreoverConfiguration.json b/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/MoreoverConfiguration.json
deleted file mode 100644
index bbbda5d..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/jsonschema/org/apache/streams/moreover/MoreoverConfiguration.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.moreover.MoreoverConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "apiKeys": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "object",
- "javaType" : "org.apache.streams.moreover.MoreoverKeyData",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "type": "string"
- },
- "key": {
- "type": "string"
- },
- "startingSequence": {
- "type": "string"
- }
- }
- }
- },
- "maxBatchSize": {
- "type": "long"
- },
- "minDelaySeconds": {
- "type": "long"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-moreover/src/main/resources/components.dot b/streams-contrib/streams-provider-moreover/src/main/resources/components.dot
deleted file mode 100644
index 93fe03e..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/resources/components.dot
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- subgraph cluster_generators {
- label="generators";
- style="dotted";
- generators_moreover [label="https://api.moreover.com"]
- }
-
- subgraph cluster_providers {
- label="providers";
- providers_moreover [label="MoreoverProvider"]
- }
-
- subgraph cluster_processors {
- label="processors";
- processors_moreover_jsonconverter [label="MoreoverJsonActivitySerializer"]
- processors_moreover_xmlconverter [label="MoreoverXmlActivitySerializer"]
- }
-
- persisters [label="persisters", shape="cylinder"];
-
- generators_moreover -> providers_moreover
- providers_moreover -> processors_moreover_jsonconverter [label="c.m.Moreover"]
- providers_moreover -> processors_moreover_xmlconverter [label="c.m.a.Article"]
-
- processors_moreover_jsonconverter -> persisters [label="o.a.s.p.j.Activity"]
- processors_moreover_xmlconverter -> persisters [label="o.a.s.p.j.Activity"]
-
-}
diff --git a/streams-contrib/streams-provider-moreover/src/main/xmlschema/com/moreover/articles10.xsd b/streams-contrib/streams-provider-moreover/src/main/xmlschema/com/moreover/articles10.xsd
deleted file mode 100644
index f95a891..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/xmlschema/com/moreover/articles10.xsd
+++ /dev/null
@@ -1,383 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<xs:schema version="1.0" xmlns:xs="http://www.w3.org/2001/XMLSchema">
-
- <xs:element name="response" type="articlesResponse"/>
-
- <xs:complexType name="articlesResponse">
- <xs:sequence>
- <xs:element name="status" type="xs:string" minOccurs="0"/>
- <xs:element name="messageCode" type="xs:string" minOccurs="0"/>
- <xs:element name="userMessage" type="xs:string" minOccurs="0"/>
- <xs:element name="developerMessage" type="xs:string" minOccurs="0"/>
- <xs:element name="articles" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="article" type="article" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="article">
- <xs:sequence>
- <xs:element name="sequenceId" type="xs:string" minOccurs="0"/>
- <xs:element name="id" type="xs:string"/>
- <xs:element name="language" type="xs:string"/>
- <xs:element name="title" type="xs:string"/>
- <xs:element name="subTitle" type="xs:string" minOccurs="0"/>
- <xs:element name="content" type="xs:string" minOccurs="0"/>
- <xs:element name="contentWithMarkup" type="xs:string" minOccurs="0"/>
- <xs:element name="tags" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="tag" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="publishedDate" type="xs:string"/>
- <xs:element name="harvestDate" type="xs:string"/>
- <xs:element name="embargoDate" type="xs:string" minOccurs="0"/>
- <xs:element name="licenseEndDate" type="xs:string"/>
- <xs:element name="url" type="xs:string"/>
- <xs:element name="originalUrl" type="xs:string" minOccurs="0"/>
- <xs:element name="commentsUrl" type="xs:string"/>
- <xs:element name="outboundUrls" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="outboundUrl" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="wordCount" type="xs:string" minOccurs="0"/>
- <xs:element name="dataFormat" type="xs:string"/>
- <xs:element name="copyright" type="xs:string"/>
- <xs:element name="loginStatus" type="xs:string"/>
- <xs:element name="duplicateGroupId" type="xs:string"/>
- <xs:element name="contentGroupIds" type="xs:string" minOccurs="0"/>
- <xs:element name="harvest" type="harvest" minOccurs="0"/>
- <xs:element name="media" type="media"/>
- <xs:element name="publishingPlatform" type="articlePublishingPlatform"/>
- <xs:element name="adultLanguage" type="xs:string"/>
- <xs:element name="topics" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="topic" type="topic" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="companies" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="company" type="company" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="locations" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="location" type="articleLocation" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="semantics" type="semantics" minOccurs="0"/>
- <xs:element name="print" type="print" minOccurs="0"/>
- <xs:element name="broadcast" type="broadcast" minOccurs="0"/>
- <xs:element name="author" type="author"/>
- <xs:element name="source" type="source"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="harvest">
- <xs:sequence>
- <xs:element name="headlineVia" type="xs:string"/>
- <xs:element name="contentVia" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="media">
- <xs:sequence>
- <xs:element name="audio" type="audioOrVideo" minOccurs="0" maxOccurs="unbounded"/>
- <xs:element name="image" type="image" minOccurs="0" maxOccurs="unbounded"/>
- <xs:element name="video" type="audioOrVideo" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="audioOrVideo">
- <xs:sequence>
- <xs:element name="url" type="xs:string"/>
- <xs:element name="mimeType" type="xs:string"/>
- <xs:element name="caption" type="xs:string"/>
- <xs:element name="duration" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="image">
- <xs:sequence>
- <xs:element name="url" type="xs:string"/>
- <xs:element name="mimeType" type="xs:string"/>
- <xs:element name="caption" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="articlePublishingPlatform">
- <xs:sequence>
- <xs:element name="itemId" type="xs:string"/>
- <xs:element name="statusId" type="xs:string"/>
- <xs:element name="inReplyToUserId" type="xs:string"/>
- <xs:element name="inReplyToStatusId" type="xs:string"/>
- <xs:element name="totalViews" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="topic">
- <xs:sequence>
- <xs:element name="name" type="xs:string" minOccurs="0"/>
- <xs:element name="group" type="xs:string" minOccurs="0"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="company">
- <xs:sequence>
- <xs:element name="name" type="xs:string"/>
- <xs:element name="symbol" type="xs:string"/>
- <xs:element name="exchange" type="xs:string"/>
- <xs:element name="isin" type="xs:string"/>
- <xs:element name="cusip" type="xs:string" minOccurs="0"/>
- <xs:element name="titleCount" type="xs:int"/>
- <xs:element name="contentCount" type="xs:int"/>
- <xs:element name="primary" type="xs:boolean"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="articleLocation">
- <xs:sequence>
- <xs:element name="name" type="xs:string"/>
- <xs:element name="type" type="xs:string"/>
- <xs:element name="class" type="xs:string"/>
- <xs:element name="mentions" type="xs:string"/>
- <xs:element name="confidence" type="xs:string"/>
- <xs:element name="country" type="country"/>
- <xs:element name="region" type="xs:string"/>
- <xs:element name="subregion" type="xs:string"/>
- <xs:element name="state" type="state"/>
- <xs:element name="latitude" type="xs:string"/>
- <xs:element name="longitude" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="country">
- <xs:sequence>
- <xs:element name="confidence" type="xs:string" minOccurs="0"/>
- <xs:element name="fipsCode" type="xs:string" minOccurs="0"/>
- <xs:element name="isoCode" type="xs:string" minOccurs="0"/>
- <xs:element name="name" type="xs:string" minOccurs="0"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="state">
- <xs:sequence>
- <xs:element name="confidence" type="xs:string" minOccurs="0"/>
- <xs:element name="fipsCode" type="xs:string" minOccurs="0"/>
- <xs:element name="name" type="xs:string" minOccurs="0"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="semantics">
- <xs:sequence>
- <xs:element name="events" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="event" type="semanticsItem" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="entities" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="entity" type="semanticsItem" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="semanticsItem">
- <xs:sequence>
- <xs:element name="properties" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="property" type="property" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="property">
- <xs:sequence>
- <xs:element name="name" type="xs:string" minOccurs="0"/>
- <xs:element name="value" type="xs:string" minOccurs="0"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="print">
- <xs:sequence>
- <xs:element name="supplement" type="xs:string"/>
- <xs:element name="publicationEdition" type="xs:string"/>
- <xs:element name="regionalEdition" type="xs:string"/>
- <xs:element name="section" type="xs:string"/>
- <xs:element name="pageNumber" type="xs:string"/>
- <xs:element name="sizeCm" type="xs:string"/>
- <xs:element name="sizePercentage" type="xs:string"/>
- <xs:element name="originLeft" type="xs:string"/>
- <xs:element name="originTop" type="xs:string"/>
- <xs:element name="width" type="xs:string"/>
- <xs:element name="height" type="xs:string"/>
- <xs:element name="byLine" type="xs:string"/>
- <xs:element name="photo" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="broadcast">
- <xs:sequence>
- <xs:element name="marketName" type="xs:string"/>
- <xs:element name="nationalNetwork" type="xs:string"/>
- <xs:element name="title" type="xs:string"/>
- <xs:element name="lines" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="line" type="line" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="line">
- <xs:sequence>
- <xs:element name="date" type="xs:string" minOccurs="0"/>
- <xs:element name="text" type="xs:string" minOccurs="0"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="author">
- <xs:sequence>
- <xs:element name="name" type="xs:string"/>
- <xs:element name="homeUrl" type="xs:string"/>
- <xs:element name="email" type="xs:string"/>
- <xs:element name="description" type="xs:string"/>
- <xs:element name="dateLastActive" type="xs:string"/>
- <xs:element name="publishingPlatform" type="authorPublishingPlatform"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="authorPublishingPlatform">
- <xs:sequence>
- <xs:element name="userName" type="xs:string"/>
- <xs:element name="userId" type="xs:string"/>
- <xs:element name="statusesCount" type="xs:string"/>
- <xs:element name="totalViews" type="xs:string"/>
- <xs:element name="followingCount" type="xs:string"/>
- <xs:element name="followersCount" type="xs:string"/>
- <xs:element name="kloutScore" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="source">
- <xs:sequence>
- <xs:element name="id" type="xs:string" minOccurs="0"/>
- <xs:element name="name" type="xs:string"/>
- <xs:element name="homeUrl" type="xs:string"/>
- <xs:element name="publisher" type="xs:string"/>
- <xs:element name="primaryLanguage" type="xs:string"/>
- <xs:element name="primaryMediaType" type="xs:string"/>
- <xs:element name="category" type="xs:string"/>
- <xs:element name="editorialRank" type="xs:string"/>
- <xs:element name="publicationId" type="xs:string" minOccurs="0"/>
- <xs:element name="channelCode" type="xs:string" minOccurs="0"/>
- <xs:element name="location" type="sourceLocation"/>
- <xs:element name="feedSource" type="xs:string" minOccurs="0"/>
- <xs:element name="feed" type="feed"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="sourceLocation">
- <xs:sequence>
- <xs:element name="country" type="xs:string"/>
- <xs:element name="countryCode" type="xs:string"/>
- <xs:element name="region" type="xs:string"/>
- <xs:element name="subregion" type="xs:string"/>
- <xs:element name="state" type="xs:string"/>
- <xs:element name="zipArea" type="xs:string"/>
- <xs:element name="zipCode" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="feed">
- <xs:sequence>
- <xs:element name="id" type="xs:string"/>
- <xs:element name="name" type="xs:string"/>
- <xs:element name="url" type="xs:string" minOccurs="0"/>
- <xs:element name="mediaType" type="xs:string"/>
- <xs:element name="publishingPlatform" type="xs:string"/>
- <xs:element name="idFromPublisher" type="xs:string"/>
- <xs:element name="generator" type="xs:string"/>
- <xs:element name="description" type="xs:string"/>
- <xs:element name="tags" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="tag" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="imageUrl" type="xs:string"/>
- <xs:element name="copyright" type="xs:string"/>
- <xs:element name="language" type="xs:string"/>
- <xs:element name="dataFormat" type="xs:string"/>
- <xs:element name="rank" type="rank"/>
- <xs:element name="inWhiteList" type="xs:string"/>
- <xs:element name="autoTopics" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="autoTopic" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="editorialTopics" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="editorialTopic" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="genre" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
- <xs:complexType name="rank">
- <xs:sequence>
- <xs:element name="autoRank" type="xs:string"/>
- <xs:element name="autoRankOrder" type="xs:string"/>
- <xs:element name="inboundLinkCount" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-</xs:schema>
-
diff --git a/streams-contrib/streams-provider-moreover/src/main/xmlschema/com/moreover/binding.xjb b/streams-contrib/streams-provider-moreover/src/main/xmlschema/com/moreover/binding.xjb
deleted file mode 100644
index d94a76b..0000000
--- a/streams-contrib/streams-provider-moreover/src/main/xmlschema/com/moreover/binding.xjb
+++ /dev/null
@@ -1,33 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<jaxb:bindings xmlns:jaxb="http://java.sun.com/xml/ns/jaxb" xmlns:xs="http://www.w3.org/2001/XMLSchema"
- xmlns:xjc="http://java.sun.com/xml/ns/jaxb/xjc" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://java.sun.com/xml/ns/jaxb http://java.sun.com/xml/ns/jaxb/bindingschema_2_0.xsd"
- version="2.1">
- <jaxb:globalBindings>
- <!-- Use java.util.Calendar instead of javax.xml.datatype.XMLGregorianCalendar for xs:dateTime -->
- <jaxb:javaType name="java.util.Calendar" xmlType="xs:dateTime"
- parseMethod="javax.xml.bind.DatatypeConverter.parseDateTime"
- printMethod="javax.xml.bind.DatatypeConverter.printDateTime" />
-
- <!-- Force all classes implements Serializable -->
- <xjc:serializable uid="1" />
- </jaxb:globalBindings>
-
-</jaxb:bindings>
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-moreover/src/site/markdown/index.md b/streams-contrib/streams-provider-moreover/src/site/markdown/index.md
deleted file mode 100644
index 3550a87..0000000
--- a/streams-contrib/streams-provider-moreover/src/site/markdown/index.md
+++ /dev/null
@@ -1,25 +0,0 @@
-streams-provider-moreover
-==============
-
-streams-provider-moreover contains schema definitions, providers, conversions, and utility classes.
-
-## Data Types
-
-| Schema |
-|--------|
-| [Moreover.json](com/moreover/Moreover.json "Moreover.json") [Moreover.json.html](apidocs/com/moreover/Moreover.html "javadoc") |
-| [articles10.xsd](com/moreover/articles10.xsd "articles10.xsd") [Moreover.xsd.html](apidocs/com/moreover/api/Article.html "javadoc") |
-
-## Configuration
-
-| Schema |
-|--------|
-| [MoreoverConfiguration.json](com/moreover/MoreoverConfiguration.json "MoreoverConfiguration.json") [MoreoverConfiguration.html](apidocs/com/moreover/MoreoverConfiguration.html "javadoc") |
-
-## Components
-
-
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-provider-moreover/src/test/java/org/apache/streams/moreover/MoreoverTestUtil.java b/streams-contrib/streams-provider-moreover/src/test/java/org/apache/streams/moreover/MoreoverTestUtil.java
deleted file mode 100644
index c9bd823..0000000
--- a/streams-contrib/streams-provider-moreover/src/test/java/org/apache/streams/moreover/MoreoverTestUtil.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.moreover;
-
-import org.apache.streams.pojo.json.Activity;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static java.util.regex.Pattern.matches;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.junit.Assert.assertThat;
-
-/**
- * MoreoverTestUtil.
- */
-public class MoreoverTestUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MoreoverTestUtil.class);
-
- /**
- *
- * @param activity
- */
- public static void validate(Activity activity) {
- assertThat(activity, is(not(nullValue())));
- assertThat(activity.getActor(), is(not(nullValue())));
- assertThat(activity.getObject(), is(not(nullValue())));
- if(activity.getObject().getId() != null) {
- assertThat(matches("id:.*:[a-z]*s:[a-zA-Z0-9]*", activity.getObject().getId()), is(true));
- }
- assertThat(activity.getObject().getObjectType(), is(not(nullValue())));
- LOGGER.debug(activity.getPublished().toString());
- }
-}
diff --git a/streams-contrib/streams-provider-moreover/src/test/java/org/apache/streams/moreover/test/MoreoverJsonActivitySerializerIT.java b/streams-contrib/streams-provider-moreover/src/test/java/org/apache/streams/moreover/test/MoreoverJsonActivitySerializerIT.java
deleted file mode 100644
index 40a46c2..0000000
--- a/streams-contrib/streams-provider-moreover/src/test/java/org/apache/streams/moreover/test/MoreoverJsonActivitySerializerIT.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.moreover.test;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.moreover.MoreoverJsonActivitySerializer;
-import org.apache.streams.moreover.MoreoverTestUtil;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.io.IOUtils;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.nio.charset.Charset;
-
-/**
- * Tests ability to serialize moreover json Strings.
- */
-public class MoreoverJsonActivitySerializerIT {
-
- JsonNode json;
- ActivitySerializer serializer = new MoreoverJsonActivitySerializer();
- ObjectMapper mapper;
-
- /**
- * Before.
- * @throws Exception Exception
- */
- @BeforeClass
- public void setup() throws Exception {
-
- StringWriter writer = new StringWriter();
- InputStream resourceAsStream = this.getClass().getResourceAsStream("/moreover.json");
- IOUtils.copy(resourceAsStream, writer, Charset.forName("UTF-8"));
-
- mapper = new ObjectMapper();
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- json = mapper.readValue(writer.toString(), JsonNode.class);
- }
-
- @Test
- public void loadData() throws Exception {
- for (JsonNode item : json) {
- MoreoverTestUtil.validate(serializer.deserialize(getString(item)));
- }
- }
-
-
- private String getString(JsonNode jsonNode) {
- try {
- return new ObjectMapper().writeValueAsString(jsonNode);
- } catch (JsonProcessingException ex) {
- throw new RuntimeException(ex);
- }
- }
-
-}
diff --git a/streams-contrib/streams-provider-moreover/src/test/java/org/apache/streams/moreover/test/MoreoverXmlActivitySerializerIT.java b/streams-contrib/streams-provider-moreover/src/test/java/org/apache/streams/moreover/test/MoreoverXmlActivitySerializerIT.java
deleted file mode 100644
index 4c69c4f..0000000
--- a/streams-contrib/streams-provider-moreover/src/test/java/org/apache/streams/moreover/test/MoreoverXmlActivitySerializerIT.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.moreover.test;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.moreover.MoreoverTestUtil;
-import org.apache.streams.moreover.MoreoverXmlActivitySerializer;
-import org.apache.streams.pojo.json.Activity;
-
-import org.apache.commons.io.IOUtils;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.nio.charset.Charset;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Tests ability to serialize moreover xml Strings
- */
-public class MoreoverXmlActivitySerializerIT {
- ActivitySerializer serializer;
- private String xml;
-
- @BeforeClass
- public void setup() throws IOException {
- serializer = new MoreoverXmlActivitySerializer();
- xml = loadXml();
- }
-
- @Test
- public void loadData() throws Exception {
- List<Activity> activities = serializer.deserializeAll(Collections.singletonList(xml));
- for (Activity activity : activities) {
- MoreoverTestUtil.validate(activity);
- }
- }
-
- private String loadXml() throws IOException {
- StringWriter writer = new StringWriter();
- InputStream resourceAsStream = this.getClass().getResourceAsStream("/moreover.xml");
- IOUtils.copy(resourceAsStream, writer, Charset.forName("UTF-8"));
- return writer.toString();
- }
-
-}
diff --git a/streams-contrib/streams-provider-moreover/src/test/java/org/apache/streams/moreover/test/provider/MoreoverProviderIT.java b/streams-contrib/streams-provider-moreover/src/test/java/org/apache/streams/moreover/test/provider/MoreoverProviderIT.java
deleted file mode 100644
index b8cf27c..0000000
--- a/streams-contrib/streams-provider-moreover/src/test/java/org/apache/streams/moreover/test/provider/MoreoverProviderIT.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.moreover.test.provider;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.moreover.MoreoverProvider;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-/**
- * Integration test for MoreoverProviderIT.
- * @Ignore this is ignored because the project doesn't have credentials to test it with during CI
- */
-public class MoreoverProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MoreoverProviderIT.class);
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Test(enabled = false)
- public void testMoreoverProvide() throws Exception {
-
- String configfile = "./target/test-classes/MoreoverProviderIT.conf";
- String outfile = "./target/test-classes/MoreoverProviderIT.stdout.txt";
-
- MoreoverProvider.main(new String[]{configfile, outfile});
-
- File out = new File(outfile);
- assert (out.exists());
- assert (out.canRead());
- assert (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- assert (outCounter.getLineNumber() >= 1);
-
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-moreover/src/test/resources/moreover.conf b/streams-contrib/streams-provider-moreover/src/test/resources/moreover.conf
deleted file mode 100644
index 3b683fe..0000000
--- a/streams-contrib/streams-provider-moreover/src/test/resources/moreover.conf
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-moreover {
- apiKeys {
- key {
- key = ""
- startingSequence = ""
- }
- }
-}
diff --git a/streams-contrib/streams-provider-rss/README.md b/streams-contrib/streams-provider-rss/README.md
deleted file mode 100644
index b10012c..0000000
--- a/streams-contrib/streams-provider-rss/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-provider-rss
-=======================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-provider-rss/pom.xml b/streams-contrib/streams-provider-rss/pom.xml
deleted file mode 100644
index 4c8adb2..0000000
--- a/streams-contrib/streams-provider-rss/pom.xml
+++ /dev/null
@@ -1,205 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-provider-rss</artifactId>
- <name>${project.artifactId}</name>
-
- <description>RSS Provider</description>
-
- <properties>
- <skipITs>true</skipITs>
- <testDataBaseURl>http://streams.peoplepattern.com.s3.amazonaws.com/test-data/</testDataBaseURl>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path-assert</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-persist-console</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.rometools</groupId>
- <artifactId>rome</artifactId>
- <version>1.7.4</version>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- </dependency>
- <dependency>
- <groupId>com.healthmarketscience.common</groupId>
- <artifactId>common-util</artifactId>
- <version>1.1.1</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.rss.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <version>1.2.1</version>
- <executions>
- <execution>
- <id>download-it-data</id>
- <phase>pre-integration-test</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
- <url>${testDataBaseURl}/${project.artifactId}.zip</url>
- <unpack>true</unpack>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- <!--<md5>df65b5642f33676313ebe4d5b69a3fff</md5>-->
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <configuration>
- <skipTests>${skipITs}</skipTests>
- </configuration>
- </plugin>
- </plugins>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes combine.children="append">
- <exclude>src/test/resources/top100.txt</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-</project>
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/processor/RssTypeConverter.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/processor/RssTypeConverter.java
deleted file mode 100644
index 5bc4cf2..0000000
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/processor/RssTypeConverter.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.rss.serializer.SyndEntryActivitySerializer;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * Converts ObjectNode representations of Rome SyndEntries to activities.
- */
-public class RssTypeConverter implements StreamsProcessor {
-
- public static final String STREAMS_ID = "RssTypeConverter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RssTypeConverter.class);
-
- private SyndEntryActivitySerializer serializer;
- private int successCount = 0;
- private int failCount = 0;
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum datum) {
- List<StreamsDatum> datums = new LinkedList<>();
- if (datum.getDocument() instanceof ObjectNode) {
- Activity activity = this.serializer.deserialize((ObjectNode) datum.getDocument());
- datums.add(new StreamsDatum(activity, activity.getId(), DateTime.now().withZone(DateTimeZone.UTC)));
- successCount ++;
- } else {
- failCount ++;
- throw new NotImplementedException("Not implemented for class type : " + datum.getDocument().getClass().toString());
-
- }
- LOGGER.debug("Processor current success count: {} and current fail: {}", successCount, failCount);
-
- return datums;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.serializer = new SyndEntryActivitySerializer();
- }
-
- @Override
- public void cleanUp() {
-
- }
-}
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java
deleted file mode 100644
index cf768e0..0000000
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.rss.serializer.SyndEntryActivitySerializer;
-import org.apache.streams.rss.serializer.SyndEntrySerializer;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.rometools.rome.feed.synd.SyndEntry;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Queue;
-import java.util.Random;
-
-/**
- * RssEventProcessor processes Rss Events.
- */
-public class RssEventProcessor implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RssEventProcessor.class);
-
- private ObjectMapper mapper = new ObjectMapper();
-
- private Queue<SyndEntry> inQueue;
- private Queue<StreamsDatum> outQueue;
-
- private Class inClass;
- private Class outClass;
-
- private SyndEntryActivitySerializer syndEntryActivitySerializer = new SyndEntryActivitySerializer();
- private SyndEntrySerializer syndEntrySerializer = new SyndEntrySerializer();
-
- public static final String TERMINATE = new String("TERMINATE");
-
- /**
- * RssEventProcessor constructor.
- * @param inQueue inQueue
- * @param outQueue outQueue
- * @param inClass inClass
- * @param outClass outClass
- */
- public RssEventProcessor(Queue<SyndEntry> inQueue, Queue<StreamsDatum> outQueue, Class inClass, Class outClass) {
- this.inQueue = inQueue;
- this.outQueue = outQueue;
- this.inClass = inClass;
- this.outClass = outClass;
- }
-
- /**
- * RssEventProcessor constructor.
- * @param inQueue inQueue
- * @param outQueue outQueue
- * @param outClass outClass
- */
- public RssEventProcessor(Queue<SyndEntry> inQueue, Queue<StreamsDatum> outQueue, Class outClass) {
- this.inQueue = inQueue;
- this.outQueue = outQueue;
- this.outClass = outClass;
- }
-
- @Override
- public void run() {
-
- while (true) {
- Object item;
- try {
- item = inQueue.poll();
- if (item instanceof String && item.equals(TERMINATE)) {
- LOGGER.info("Terminating!");
- break;
- }
-
- Thread.sleep(new Random().nextInt(100));
-
- // if the target is string, just pass-through
- if ( String.class.equals(outClass)) {
- outQueue.offer(new StreamsDatum(item.toString()));
- } else if ( SyndEntry.class.equals(outClass)) {
- outQueue.offer(new StreamsDatum(item));
- } else if ( Activity.class.equals(outClass)) {
- // convert to desired format
- SyndEntry entry = (SyndEntry)item;
- if ( entry != null ) {
- Activity out = syndEntryActivitySerializer.deserialize(this.syndEntrySerializer.deserialize((SyndEntry)item));
-
- if ( out != null ) {
- outQueue.offer(new StreamsDatum(out));
- }
- }
- }
-
- } catch (Exception ex) {
- ex.printStackTrace();
- }
- }
- }
-
-}
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProvider.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProvider.java
deleted file mode 100644
index 81b4f79..0000000
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProvider.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.rss.RssStreamConfiguration;
-import org.apache.streams.rss.provider.perpetual.RssFeedScheduler;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.math.BigInteger;
-import java.util.Queue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * RSS {@link org.apache.streams.core.StreamsProvider} that provides content from rss feeds in boilerpipe format
- */
-public class RssStreamProvider implements StreamsProvider {
-
- private static final String STREAMS_ID = "RssStreamProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RssStreamProvider.class);
-
- private static final int MAX_SIZE = 1000;
-
- private RssStreamConfiguration config;
- private boolean perpetual;
- private ExecutorService executor;
- private BlockingQueue<StreamsDatum> dataQueue;
- private AtomicBoolean isComplete;
-
- @VisibleForTesting
- protected RssFeedScheduler scheduler;
-
- public RssStreamProvider() {
- this(new ComponentConfigurator<>(RssStreamConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("rss")), false);
- }
-
- public RssStreamProvider(boolean perpetual) {
- this(new ComponentConfigurator<>(RssStreamConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("rss")), perpetual);
- }
-
- public RssStreamProvider(RssStreamConfiguration config) {
- this(config, false);
- }
-
- public RssStreamProvider(RssStreamConfiguration config, boolean perpetual) {
- this.perpetual = perpetual;
- this.config = config;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
- LOGGER.trace("Starting Rss Scheduler");
- this.executor.submit(this.scheduler);
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- Queue<StreamsDatum> batch = new ConcurrentLinkedQueue<>();
- int batchSize = 0;
- while (!this.dataQueue.isEmpty() && batchSize < MAX_SIZE) {
- StreamsDatum datum = ComponentUtils.pollWhileNotEmpty(this.dataQueue);
- if (datum != null) {
- ++batchSize;
- batch.add(datum);
- }
- }
- this.isComplete.set(this.scheduler.isComplete() && batch.isEmpty() && this.dataQueue.isEmpty());
- return new StreamsResultSet(batch);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return !this.isComplete.get();
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.executor = new ThreadPoolExecutor(1, 4, 15L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
- this.dataQueue = new LinkedBlockingQueue<>();
- this.scheduler = getScheduler(this.dataQueue);
- this.isComplete = new AtomicBoolean(false);
- int consecutiveEmptyReads = 0;
- }
-
- @VisibleForTesting
- protected RssFeedScheduler getScheduler(BlockingQueue<StreamsDatum> queue) {
- if (this.perpetual) {
- return new RssFeedScheduler(this.executor, this.config.getFeeds(), queue);
- } else {
- return new RssFeedScheduler(this.executor, this.config.getFeeds(), queue, 0);
- }
- }
-
- @Override
- public void cleanUp() {
- this.scheduler.stop();
- ComponentUtils.shutdownExecutor(this.executor, 10, 10);
- }
-
- /**
- * To use from command line:
- *
- * <p/>
- * Supply configuration similar to src/test/resources/rss.conf
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.rss.provider.RssStreamProvider -Dexec.args="rss.conf articles.json"
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- RssStreamConfiguration config = new ComponentConfigurator<>(RssStreamConfiguration.class).detectConfiguration(typesafe, "rss");
- RssStreamProvider provider = new RssStreamProvider(config);
-
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- try {
- json = mapper.writeValueAsString(datum.getDocument());
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-}
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java
deleted file mode 100644
index 8437ee0..0000000
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.data.util.RFC3339Utils;
-import org.apache.streams.rss.serializer.SyndEntrySerializer;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.annotations.VisibleForTesting;
-import com.rometools.rome.feed.synd.SyndEntry;
-import com.rometools.rome.feed.synd.SyndFeed;
-import com.rometools.rome.io.FeedException;
-import com.rometools.rome.io.SyndFeedInput;
-
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * A {@link java.lang.Runnable} task that queues rss feed data.
- *
- * <p/>
- * <code>RssStreamProviderTask</code> reads the content of an rss feed and queues the articles from
- * the feed inform of a {@link com.fasterxml.jackson.databind.node.ObjectNode} wrapped in a {@link org.apache.streams.core.StreamsDatum}.
- * The task can filter articles by a published date. If the task cannot parse the date of the article or the article does not contain a
- * published date, by default the task will attempt to queue article.
- *
- * <p/>
- * A task can be run in perpetual mode which will store the article urls in a static variable. The next time a
- * <code>RssStreamProviderTask</code> is run, it will not queue data that was seen the previous time the rss feed was read.
- * This is an attempt to reduce multiple copies of an article from being output by a
- * {@link org.apache.streams.rss.provider.RssStreamProvider}.
- *
- * <p/>
- * ** Warning! **
- * It still is possible to output multiples of the same article. If multiple tasks executions for the same rss feed overlap
- * in execution time, it possible that the previously seen articles static variable will not have been updated in time.
- *
- */
-public class RssStreamProviderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RssStreamProviderTask.class);
- private static final int DEFAULT_TIME_OUT = 10000; // 10 seconds
- private static final String RSS_KEY = "rssFeed";
- private static final String URI_KEY = "uri";
- private static final String LINK_KEY = "link";
- private static final String DATE_KEY = "publishedDate";
-
- /**
- * Map that contains the Set of previously seen articles by an rss feed.
- */
- @VisibleForTesting
- protected Map<String, Set<String>> PREVIOUSLY_SEEN = new ConcurrentHashMap<>();
-
-
- private BlockingQueue<StreamsDatum> dataQueue;
- private String rssFeed;
- private int timeOut;
- private SyndEntrySerializer serializer;
- private DateTime publishedSince;
- private boolean perpetual;
-
-
- /**
- * Non-perpetual mode, no date filter, time out of 10 sec
- * @see {@link org.apache.streams.rss.provider.RssStreamProviderTask
- * #RssStreamProviderTask(java.util.concurrent.BlockingQueue, String, org.joda.time.DateTime, int, boolean)}
- * @param queue queue
- * @param rssFeed rssFeed
- */
- public RssStreamProviderTask(BlockingQueue<StreamsDatum> queue, String rssFeed) {
- this(queue, rssFeed, new DateTime().minusYears(30), DEFAULT_TIME_OUT, false);
- }
-
- /**
- * Non-perpetual mode, no date filter.
- * @see {@link org.apache.streams.rss.provider.RssStreamProviderTask
- * #RssStreamProviderTask(java.util.concurrent.BlockingQueue, String, org.joda.time.DateTime, int, boolean)}
- * @param queue queue
- * @param rssFeed rssFeed
- * @param timeOut timeOut
- */
- public RssStreamProviderTask(BlockingQueue<StreamsDatum> queue, String rssFeed, int timeOut) {
- this(queue, rssFeed, new DateTime().minusYears(30), timeOut, false);
- }
-
- /**
- * Non-perpetual mode, time out of 10 sec
- * @see {@link org.apache.streams.rss.provider.RssStreamProviderTask
- * #RssStreamProviderTask(java.util.concurrent.BlockingQueue, String, org.joda.time.DateTime, int, boolean)}
- * @param queue queue
- * @param rssFeed rssFeed
- * @param publishedSince publishedSince
- */
- public RssStreamProviderTask(BlockingQueue<StreamsDatum> queue, String rssFeed, DateTime publishedSince) {
- this(queue, rssFeed, publishedSince, DEFAULT_TIME_OUT, false);
- }
-
- /**
- * RssStreamProviderTask that reads an rss feed url and queues the resulting articles as StreamsDatums with the documents
- * being object nodes.
- * @param queue Queue to push data to
- * @param rssFeed url of rss feed to read
- * @param publishedSince DateTime to filter articles by, will queue articles with published times after this
- * @param timeOut url connection timeout in milliseconds
- * @param perpetual true, if you want to run in perpetual mode. NOT RECOMMENDED
- */
- public RssStreamProviderTask(BlockingQueue<StreamsDatum> queue, String rssFeed, DateTime publishedSince, int timeOut, boolean perpetual) {
- this.dataQueue = queue;
- this.rssFeed = rssFeed;
- this.timeOut = timeOut;
- this.publishedSince = publishedSince;
- this.serializer = new SyndEntrySerializer();
- this.perpetual = perpetual;
- }
-
- /**
- * The rss feed url that this task is responsible for reading.
- * @return rss feed url
- */
- public String getRssFeed() {
- return this.rssFeed;
- }
-
- @Override
- public void run() {
- try {
- Set<String> batch = queueFeedEntries(new URL(this.rssFeed));
- if (this.perpetual) {
- PREVIOUSLY_SEEN.put(this.getRssFeed(), batch);
- }
- } catch (IOException | FeedException ex) {
- LOGGER.warn("Exception while reading rss stream, {} : {}", this.rssFeed, ex);
- }
- }
-
- /**
- * Reads the url and queues the data
- * @param feedUrl rss feed url
- * @return set of all article urls that were read from the feed
- * @throws IOException when it cannot connect to the url or the url is malformed
- * @throws FeedException when it cannot reed the feed.
- */
- @VisibleForTesting
- protected Set<String> queueFeedEntries(URL feedUrl) throws IOException, FeedException {
-
- // ConcurrentHashSet is preferable, but it's only in guava 15+
- // spark 1.5.0 uses guava 14 so for the moment this is the workaround
- Set<String> batch = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
- URLConnection connection = feedUrl.openConnection();
- connection.setConnectTimeout(this.timeOut);
- connection.setConnectTimeout(this.timeOut);
- SyndFeedInput input = new SyndFeedInput();
- SyndFeed feed = input.build(new InputStreamReader(connection.getInputStream()));
- for (Object entryObj : feed.getEntries()) {
- SyndEntry entry = (SyndEntry) entryObj;
- ObjectNode nodeEntry = this.serializer.deserialize(entry);
- nodeEntry.put(RSS_KEY, this.rssFeed);
- String entryId = determineId(nodeEntry);
- batch.add(entryId);
- StreamsDatum datum = new StreamsDatum(nodeEntry);
- try {
- JsonNode published = nodeEntry.get(DATE_KEY);
- if (published != null) {
- try {
- DateTime date = RFC3339Utils.parseToUTC(published.asText());
- if (date.isAfter(this.publishedSince) && (!this.perpetual || !seenBefore(entryId, this.rssFeed))) {
- this.dataQueue.put(datum);
- LOGGER.debug("Added entry, {}, to provider queue.", entryId);
- }
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- } catch (Exception ex) {
- LOGGER.trace("Failed to parse date from object node, attempting to add node to queue by default.");
- if (!this.perpetual || !seenBefore(entryId, this.rssFeed)) {
- this.dataQueue.put(datum);
- LOGGER.debug("Added entry, {}, to provider queue.", entryId);
- }
- }
- } else {
- LOGGER.debug("No published date present, attempting to add node to queue by default.");
- if (!this.perpetual || !seenBefore(entryId, this.rssFeed)) {
- this.dataQueue.put(datum);
- LOGGER.debug("Added entry, {}, to provider queue.", entryId);
- }
- }
- } catch (InterruptedException ie) {
- LOGGER.error("Interupted Exception.");
- Thread.currentThread().interrupt();
- }
- }
- return batch;
- }
-
- /**
- * Returns link to the article to use as the id.
- * @param node node
- * @return String
- */
- private String determineId(ObjectNode node) {
- String id = null;
- if (node.get(URI_KEY) != null && !node.get(URI_KEY).textValue().equals("")) {
- id = node.get(URI_KEY).textValue();
- } else if (node.get(LINK_KEY) != null && !node.get(LINK_KEY).textValue().equals("")) {
- id = node.get(LINK_KEY).textValue();
- }
- return id;
- }
-
- /**
- * Returns false if the artile was previously seen in another task for this feed.
- * @param id id
- * @param rssFeed rssFeed
- * @return boolean seenBefore
- */
- private boolean seenBefore(String id, String rssFeed) {
- Set<String> previousBatch = PREVIOUSLY_SEEN.get(rssFeed);
- if (previousBatch == null) {
- return false;
- }
- return previousBatch.contains(id);
- }
-
-
-}
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/perpetual/RssFeedScheduler.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/perpetual/RssFeedScheduler.java
deleted file mode 100644
index 975749f..0000000
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/perpetual/RssFeedScheduler.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.provider.perpetual;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.rss.FeedDetails;
-import org.apache.streams.rss.provider.RssStreamProviderTask;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * RssFeedScheduler launches threads to collect data from rss feeds.
- */
-public class RssFeedScheduler implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RssFeedScheduler.class);
- private static final int DEFAULT_PEROID = 10; // 1 minute
-
- private ExecutorService service;
- private List<FeedDetails> feedDetailsList;
- private int peroid;
- private AtomicBoolean keepRunning;
- private AtomicBoolean complete;
- private Map<String, Long> lastScheduled;
- private BlockingQueue<StreamsDatum> dataQueue;
-
- public RssFeedScheduler(ExecutorService service, List<FeedDetails> feedDetailsList, BlockingQueue<StreamsDatum> dataQueue) {
- this(service, feedDetailsList, dataQueue, DEFAULT_PEROID);
- }
-
- /**
- * RssFeedScheduler constructor.
- * @param service service
- * @param feedDetailsList feedDetailsList
- * @param dataQueue dataQueue
- * @param peroid peroid
- */
- public RssFeedScheduler(ExecutorService service, List<FeedDetails> feedDetailsList, BlockingQueue<StreamsDatum> dataQueue, int peroid) {
- this.service = service;
- this.feedDetailsList = feedDetailsList;
- this.peroid = peroid;
- this.keepRunning = new AtomicBoolean(true);
- this.lastScheduled = new HashMap<>();
- this.dataQueue = dataQueue;
- this.complete = new AtomicBoolean(false);
- }
-
- public void stop() {
- this.keepRunning.set(false);
- }
-
- public boolean isComplete() {
- return this.complete.get();
- }
-
- @Override
- public void run() {
- this.complete.set(false);
- try {
- if (this.peroid <= 0) {
- scheduleFeeds();
- } else {
- while (this.keepRunning.get()) {
- scheduleFeeds();
- Thread.sleep(this.peroid * 60000);
- }
- }
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- } finally {
- this.service = null;
- LOGGER.info("{} completed scheduling of feeds.", this.getClass().getName());
- this.complete.set(true);
- }
- }
-
- /**
- * Schedule Feeds.
- */
- public void scheduleFeeds() {
- for (FeedDetails detail : this.feedDetailsList) {
- Long lastTime = null;
- if ((lastTime = this.lastScheduled.get(detail.getUrl())) == null) {
- lastTime = 0L;
- }
- long currentTime = System.currentTimeMillis();
- long pollInterval;
- if (detail.getPollIntervalMillis() == null) {
- pollInterval = 0;
- } else {
- pollInterval = detail.getPollIntervalMillis();
- }
- if (currentTime - lastTime > pollInterval) {
- this.service.execute(new RssStreamProviderTask(this.dataQueue, detail.getUrl()));
- LOGGER.trace("Scheduled data collection on rss feed, {}", detail.getUrl());
- this.lastScheduled.put(detail.getUrl(), currentTime);
- }
- }
- }
-}
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java
deleted file mode 100644
index 45d5d96..0000000
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntryActivitySerializer.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.serializer;
-
-import org.apache.streams.data.ActivitySerializer;
-import org.apache.streams.data.util.RFC3339Utils;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Author;
-import org.apache.streams.pojo.json.Provider;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Objects;
-
-public class SyndEntryActivitySerializer implements ActivitySerializer<ObjectNode> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SyndEntryActivitySerializer.class);
-
- private boolean includeRomeExtension;
-
- public SyndEntryActivitySerializer() {
- this(true);
- }
-
- public SyndEntryActivitySerializer(boolean includeRomeExtension) {
- this.includeRomeExtension = includeRomeExtension;
- }
-
- @Override
- public List<Activity> deserializeAll(List<ObjectNode> objectNodes) {
- List<Activity> result = new LinkedList<>();
- for (ObjectNode node : objectNodes) {
- result.add(deserialize(node));
- }
- return result;
- }
-
- @Override
- public String serializationFormat() {
- return "application/streams-provider-rss";
- }
-
- @Override
- public ObjectNode serialize(Activity deserialized) {
- throw new UnsupportedOperationException("Cannot currently serialize to Rome");
- }
-
- @Override
- public Activity deserialize(ObjectNode syndEntry) {
- return deserializeWithRomeExtension(syndEntry, this.includeRomeExtension);
- }
-
- /**
- * deserializeWithRomeExtension ObjectNode entry withExtension.
- * @param entry ObjectNode
- * @param withExtension whether to add Rome Extension
- * @return Activity
- */
- public Activity deserializeWithRomeExtension(ObjectNode entry, boolean withExtension) {
- Objects.requireNonNull(entry);
-
- Activity activity = new Activity();
- Provider provider = buildProvider(entry);
- ActivityObject actor = buildActor(entry);
- ActivityObject activityObject = buildActivityObject(entry);
-
- activityObject.setUrl(provider.getUrl());
- activityObject.setAuthor(actor.getAuthor());
-
- activity.setUrl(provider.getUrl());
- activity.setProvider(provider);
- activity.setActor(actor);
- activity.setVerb("post");
- activity.setId("id:rss:post:" + activity.getUrl());
-
- JsonNode published = entry.get("publishedDate");
- if (published != null) {
- try {
- activity.setPublished(RFC3339Utils.parseToUTC(published.textValue()));
- } catch (Exception ex) {
- LOGGER.warn("Failed to parse date : {}", published.textValue());
-
- DateTime now = DateTime.now().withZone(DateTimeZone.UTC);
- activity.setPublished(now);
- }
- }
-
- activity.setUpdated(activityObject.getUpdated());
- activity.setObject(activityObject);
-
- if (withExtension) {
- activity = addRomeExtension(activity, entry);
- }
-
- return activity;
- }
-
- /**
- * Given an RSS entry, extra out the author and actor information and return it
- * in an actor object
- *
- * @param entry entry
- * @return $.actor
- */
- private ActivityObject buildActor(ObjectNode entry) {
- ActivityObject actor = new ActivityObject();
- Author author = new Author();
-
- if (entry.get("author") != null) {
- author.setId(entry.get("author").textValue());
- author.setDisplayName(entry.get("author").textValue());
-
- actor.setAuthor(author);
- String uriToSet = entry.get("rssFeed") != null ? entry.get("rssFeed").asText() : null;
-
- actor.setId("id:rss:" + uriToSet + ":" + author.getId());
- actor.setDisplayName(author.getDisplayName());
- }
-
- return actor;
- }
-
- /**
- * Given an RSS object, build the ActivityObject.
- *
- * @param entry ObjectNode
- * @return $.object
- */
- private ActivityObject buildActivityObject(ObjectNode entry) {
- ActivityObject activityObject = new ActivityObject();
-
- JsonNode summary = entry.get("description");
- if (summary != null) {
- activityObject.setSummary(summary.textValue());
- } else if ((summary = entry.get("title")) != null) {
- activityObject.setSummary(summary.textValue());
- }
-
- return activityObject;
- }
-
- /**
- * Given an RSS object, build and return the Provider object.
- *
- * @param entry ObjectNode
- * @return $.provider
- */
- private Provider buildProvider(ObjectNode entry) {
- Provider provider = new Provider();
-
- String link = null;
- String uri = null;
- String resourceLocation = null;
-
- if (entry.get("link") != null) {
- link = entry.get("link").textValue();
- }
- if (entry.get("uri") != null) {
- uri = entry.get("uri").textValue();
- }
- /*
- * Order of precedence for resourceLocation selection
- *
- * 1. Valid URI
- * 2. Valid Link
- * 3. Non-null URI
- * 4. Non-null Link
- */
- if (isValidResource(uri)) {
- resourceLocation = uri;
- } else if (isValidResource(link)) {
- resourceLocation = link;
- } else if (uri != null || link != null) {
- resourceLocation = (uri != null) ? uri : link;
- }
-
- provider.setId("id:providers:rss");
- provider.setUrl(resourceLocation);
- provider.setDisplayName("RSS");
-
- return provider;
- }
-
- /**
- * Tests whether or not the passed in resource is a valid URI.
- * @param resource resource
- * @return boolean of whether or not the resource is valid
- */
- private boolean isValidResource(String resource) {
- return resource != null && (resource.startsWith("http") || resource.startsWith("www"));
- }
-
- /**
- * Given an RSS object and an existing activity,
- * add the Rome extension to that activity and return it.
- *
- * @param activity Activity
- * @param entry ObjectNode
- * @return Activity
- */
- private Activity addRomeExtension(Activity activity, ObjectNode entry) {
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- ObjectNode activityRoot = mapper.convertValue(activity, ObjectNode.class);
- ObjectNode extensions = JsonNodeFactory.instance.objectNode();
-
- extensions.put("rome", entry);
- activityRoot.put("extensions", extensions);
-
- activity = mapper.convertValue(activityRoot, Activity.class);
-
- return activity;
- }
-}
diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java
deleted file mode 100644
index 1b84d0a..0000000
--- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java
+++ /dev/null
@@ -1,327 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.serializer;
-
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.rometools.rome.feed.module.Module;
-import com.rometools.rome.feed.rss.Enclosure;
-import com.rometools.rome.feed.synd.SyndContent;
-import com.rometools.rome.feed.synd.SyndEnclosure;
-import com.rometools.rome.feed.synd.SyndEntry;
-import com.rometools.rome.feed.synd.SyndFeed;
-import com.rometools.rome.feed.synd.SyndImage;
-import com.rometools.rome.feed.synd.SyndLinkImpl;
-
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Date;
-import java.util.List;
-
-/**
- * Since SyndEntry is not Serializable, we cannot emit them from any StreamOperation. So the CommunityRssProvider
- * converts the SyndEntries to ObjectNodes using this class.
- */
-public class SyndEntrySerializer {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SyndEntrySerializer.class);
-
- public ObjectNode deserialize(SyndEntry entry) {
- return deserializeRomeEntry(entry);
- }
-
- private ObjectNode deserializeRomeEntry(SyndEntry entry) {
- JsonNodeFactory factory = JsonNodeFactory.instance;
- ObjectNode root = factory.objectNode();
-
- serializeString(entry.getAuthor(), "author", root);
- serializeListOfStrings(entry.getAuthors(), "authors", root, factory);
- serializeCategories(root, factory, entry.getCategories());
- serializeContents(root, factory, entry.getContents());
- serializeListOfStrings(entry.getContributors(), "contributors", root, factory);
- serializeDescription(root, factory, entry.getDescription());
- serializeEnclosures(root, factory, entry.getEnclosures());
- serializeForeignMarkUp(root, factory, entry.getForeignMarkup());
- serializeString(entry.getLink(), "link", root);
- serializeLinks(root, factory, entry.getLinks());
- serializeModules(root, factory, entry.getModules());
- serializeDate(root, entry.getPublishedDate(), "publishedDate");
- serializeSource(root, factory, entry.getSource());
- serializeString(entry.getTitle(), "title", root);
- serializeDate(root, entry.getUpdatedDate(), "updateDate");
- serializeString(entry.getUri(), "uri", root);
-
- return root;
- }
-
-
- private void serializeCategories(ObjectNode root, JsonNodeFactory factory, List categories) {
- if (categories == null || categories.size() == 0) {
- return;
- }
- ArrayNode cats = factory.arrayNode();
- for (Object obj : categories) {
- if (obj instanceof com.rometools.rome.feed.rss.Category) {
- ObjectNode catNode = factory.objectNode();
- com.rometools.rome.feed.rss.Category category = (com.rometools.rome.feed.rss.Category) obj;
- if (category.getDomain() != null) {
- catNode.put("domain", category.getDomain());
- }
- if (category.getValue() != null) {
- catNode.put("value", category.getValue());
- }
- cats.add(catNode);
- } else if (obj instanceof com.rometools.rome.feed.atom.Category) {
- com.rometools.rome.feed.atom.Category category = (com.rometools.rome.feed.atom.Category) obj;
- ObjectNode catNode = factory.objectNode();
- if (category.getLabel() != null) {
- catNode.put("label", category.getLabel());
- }
- if (category.getScheme() != null) {
- catNode.put("scheme", category.getScheme());
- }
- if (category.getSchemeResolved() != null) {
- catNode.put("schemeResolved", category.getSchemeResolved());
- }
- if (category.getTerm() != null ) {
- catNode.put("term", category.getTerm());
- }
- cats.add(catNode);
- }
- }
- root.put("categories", cats);
- }
-
- private void serializeContents(ObjectNode root, JsonNodeFactory factory, List contents) {
- if (contents == null || contents.size() == 0) {
- return;
- }
- ArrayNode contentsArray = factory.arrayNode();
- for (Object obj : contents) {
- ObjectNode content = factory.objectNode();
- if (obj instanceof com.rometools.rome.feed.rss.Content) {
- com.rometools.rome.feed.rss.Content rssContent = (com.rometools.rome.feed.rss.Content) obj;
- content.put("type", rssContent.getType());
- content.put("value", rssContent.getValue());
- }
- if (obj instanceof com.rometools.rome.feed.atom.Content) {
- com.rometools.rome.feed.atom.Content atomContent = (com.rometools.rome.feed.atom.Content) obj;
- content.put("type", atomContent.getType());
- content.put("value", atomContent.getValue());
- content.put("mode", atomContent.getMode());
- content.put("src", atomContent.getSrc());
- }
- contentsArray.add(content);
- }
- root.put("contents", contentsArray);
- }
-
- private void serializeDate(ObjectNode root, Date date, String key) {
- DateTimeFormatter formatter = ISODateTimeFormat.dateTime();
- if (date == null) {
- return;
- }
- root.put(key, formatter.print(date.getTime()));
- }
-
- private void serializeDescription(ObjectNode root, JsonNodeFactory factory, SyndContent synd) {
- if (synd == null) {
- return;
- }
- ObjectNode content = factory.objectNode();
- if (synd.getValue() != null) {
- content.put("value", synd.getValue());
- }
- if (synd.getMode() != null) {
- content.put("mode", synd.getMode());
- }
- if (synd.getType() != null) {
- content.put("type", synd.getType());
- }
- root.put("description", content);
- }
-
- private void serializeEnclosures(ObjectNode root, JsonNodeFactory factory, List enclosures) {
- if (enclosures == null || enclosures.size() == 0) {
- return;
- }
- ArrayNode encls = factory.arrayNode();
- for (Object obj : enclosures) {
- if (obj instanceof Enclosure) {
- Enclosure enclosure = (Enclosure) obj;
- ObjectNode encl = factory.objectNode();
- if (enclosure.getType() != null) {
- encl.put("type", enclosure.getType());
- }
- if (enclosure.getUrl() != null) {
- encl.put("url", enclosure.getUrl());
- }
- encl.put("length", enclosure.getLength());
- encls.add(encl);
- } else if (obj instanceof SyndEnclosure) {
- SyndEnclosure enclosure = (SyndEnclosure) obj;
- ObjectNode encl = factory.objectNode();
- if (enclosure.getType() != null) {
- encl.put("type", enclosure.getType());
- }
- if (enclosure.getUrl() != null) {
- encl.put("url", enclosure.getUrl());
- }
- encl.put("length", enclosure.getLength());
- encls.add(encl);
- } else {
- LOGGER.warn("serializeEnclosures does not handle type : {}", obj.getClass().toString());
- }
- }
- root.put("enclosures", encls);
- }
-
- private void serializeForeignMarkUp(ObjectNode root, JsonNodeFactory factory, Object foreignMarkUp) {
- if (foreignMarkUp == null) {
- return;
- }
- if (foreignMarkUp instanceof String) {
- root.put("foreignEnclosures", (String) foreignMarkUp);
- } else if (foreignMarkUp instanceof List) {
- List foreignList = (List) foreignMarkUp;
- if (foreignList.size() == 0) {
- return;
- }
- if (foreignList.get(0) instanceof String) {
- serializeListOfStrings(foreignList, "foreignEnclosures", root, factory);
- } else {
- LOGGER.debug("SyndEntry.getForeignMarkUp is not of type String. Need to handle the case of class : {}",
- ((List)foreignMarkUp).get(0).getClass().toString());
- }
- } else {
- LOGGER.debug("SyndEntry.getForeignMarkUp is not of an expected type. Need to handle the case of class : {}",
- foreignMarkUp.getClass().toString());
- }
- }
-
- private void serializeImage(ObjectNode root, JsonNodeFactory factory, SyndImage image) {
- if (image == null) {
- return;
- }
- ObjectNode imageNode = factory.objectNode();
- serializeString(image.getDescription(), "description", imageNode);
- serializeString(image.getLink(), "link", imageNode);
- serializeString(image.getUrl(), "url", imageNode);
- serializeString(image.getTitle(), "title", imageNode);
- root.put("image", imageNode);
- }
-
- private void serializeListOfStrings(List toSerialize, String key, ObjectNode node, JsonNodeFactory factory) {
- if (toSerialize == null || toSerialize.size() == 0) {
- return;
- }
- ArrayNode keyNode = factory.arrayNode();
- for (Object obj : toSerialize) {
- if (obj instanceof String) {
- keyNode.add((String) obj);
- } else {
- LOGGER.debug("Array at Key:{} was expecting item types of String. Received class : {}", key, obj.getClass().toString());
- }
- }
- node.put(key, keyNode);
- }
-
- private void serializeLinks(ObjectNode root, JsonNodeFactory factory, List links) {
- if( links.size() == 0 ) {
- root.put("links", factory.arrayNode());
- } else if (links.get(0) instanceof String) {
- serializeListOfStrings(links, "links", root, factory);
- } else if (links.get(0) instanceof SyndLinkImpl) {
- ArrayNode linksArray = factory.arrayNode();
- SyndLinkImpl syndLink;
- ObjectNode linkNode;
- for (Object obj : links) {
- linkNode = factory.objectNode();
- syndLink = (SyndLinkImpl) obj;
- linkNode.put("rel", syndLink.getRel());
- linkNode.put("href", syndLink.getHref());
- linkNode.put("type", syndLink.getType());
- linkNode.put("length", syndLink.getLength());
- linkNode.put("hrefLang", syndLink.getHreflang());
- linkNode.put("title", syndLink.getTitle());
- linksArray.add(linkNode);
- }
- root.put("links", linksArray);
- } else {
- LOGGER.error("No implementation for handling links of class : {}", links.get(0).getClass().toString());
- }
- }
-
- private void serializeModules(ObjectNode root, JsonNodeFactory factory, List modules) {
- if (modules == null || modules.size() == 0) {
- return;
- }
- ArrayNode modulesArray = factory.arrayNode();
- for (Object obj : modules) {
- if (obj instanceof Module) {
- Module mod = (Module) obj;
- if (mod.getUri() != null) {
- modulesArray.add(mod.getUri());
- }
- } else {
- LOGGER.debug("SyndEntry.getModules() items are not of type Module. Need to handle the case of class : {}",
- obj.getClass().toString());
- }
- }
- root.put("modules", modulesArray);
- }
-
- private void serializeSource(ObjectNode root, JsonNodeFactory factory, SyndFeed source) {
- if (source == null) {
- return;
- }
- ObjectNode sourceNode = factory.objectNode();
- serializeString(source.getAuthor(), "author", sourceNode);
- serializeListOfStrings(source.getAuthors(), "authors", sourceNode, factory);
- serializeCategories(sourceNode, factory, source.getCategories());
- serializeString(source.getCopyright(), "copyright", sourceNode);
- serializeListOfStrings(source.getContributors(), "contributors", sourceNode, factory);
- serializeString(source.getDescription(), "description", sourceNode);
- serializeDescription(sourceNode, factory, source.getDescriptionEx());
- // source.getEntries(); wtf?
- serializeString(source.getFeedType(), "feedType", sourceNode);
- serializeImage(sourceNode, factory, source.getImage());
- serializeForeignMarkUp(sourceNode, factory, source.getForeignMarkup());
- serializeString(source.getLanguage(), "language", sourceNode);
- serializeString(source.getLink(), "link", sourceNode);
- serializeListOfStrings(source.getLinks(), "links", sourceNode, factory);
- serializeModules(sourceNode, factory, source.getModules());
- serializeDate(sourceNode, source.getPublishedDate(), "publishedDate");
- serializeString(source.getTitle(), "title", sourceNode);
- serializeString(source.getUri(), "uri", sourceNode);
-
- root.put("source", sourceNode);
- }
-
- private void serializeString(String string, String key, ObjectNode node) {
- if (string != null && !string.equals("")) {
- node.put(key, string);
- }
- }
-
-}
diff --git a/streams-contrib/streams-provider-rss/src/main/jsonschema/RssStreamConfiguration.json b/streams-contrib/streams-provider-rss/src/main/jsonschema/RssStreamConfiguration.json
deleted file mode 100644
index 5365aed..0000000
--- a/streams-contrib/streams-provider-rss/src/main/jsonschema/RssStreamConfiguration.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.rss.RssStreamConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "feeds": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "object",
- "javaType" : "org.apache.streams.rss.FeedDetails",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "url": {
- "type": "string",
- "description": "The feed url"
- },
- "pollIntervalMillis": {
- "type": "integer",
- "description": "Polling interval in ms"
- }
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-rss/src/main/resources/components.dot b/streams-contrib/streams-provider-rss/src/main/resources/components.dot
deleted file mode 100644
index 429004d..0000000
--- a/streams-contrib/streams-provider-rss/src/main/resources/components.dot
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- subgraph cluster_generators {
- label="generators";
- style="dotted";
- generators_rss [label="RSS Feeds"]
- }
-
- subgraph cluster_providers {
- label="providers";
- providers_rss_streamprovider [label="RssStreamProvider"]
- }
-
- subgraph cluster_processors {
- label="processors";
- processors_rss_activity [label="RssTypeConverter"]
- }
-
- persisters [label="persisters", shape="cylinder"];
-
- generators_rss -> providers_rss_streamprovider
- providers_rss_streamprovider -> processors_rss_activity [label="c.s.s.f.s.SyndEntry"]
-
- processors_rss_activity -> persisters [label="o.a.s.p.j.Activity"]
-
-}
diff --git a/streams-contrib/streams-provider-rss/src/site/markdown/index.md b/streams-contrib/streams-provider-rss/src/site/markdown/index.md
deleted file mode 100644
index a2d0638..0000000
--- a/streams-contrib/streams-provider-rss/src/site/markdown/index.md
+++ /dev/null
@@ -1,27 +0,0 @@
-streams-provider-rss
-====================
-
-## Data Types
-
-| Schema |
-|--------|
-| [contents.xsd](http://www.dlese.org/Metadata/opml/2.0/contents.xsd "contents.xsd") [FeedDetails.html](apidocs/org/apache/streams/rss/FeedDetails.html "javadoc") |
-
-## Components
-
-
-
-| Class | Configuration | Example Configuration(s) |
-|-------|---------------|--------------------------|
-| RssStreamProvider [RssStreamProvider.html](apidocs/org/apache/streams/rss/provider/RssStreamProvider.html "javadoc") | [RssStreamConfiguration.json](RssStreamConfiguration.json "RssStreamConfiguration.json") [RssStreamConfiguration.html](apidocs/org/apache/streams/rss/RssStreamConfiguration.html "javadoc") | [rss.conf](rss.conf "rss.conf") |
-
-Test:
------
-
-Build with integration testing enabled
-
- mvn clean test verify -DskipITs=false
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java
deleted file mode 100644
index 564e215..0000000
--- a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.provider;
-
-import org.apache.streams.core.StreamsDatum;
-
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.testng.annotations.Test;
-
-import java.net.URL;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-/**
- * Unit tests for {@link org.apache.streams.rss.provider.RssStreamProviderTask}
- */
-public class RssStreamProviderTaskIT {
-
- /**
- * Test that a task can read a valid rss from a url and queue the data.
- * @throws Exception Exception
- */
- @Test
- public void testNonPerpetualNoTimeFramePull() throws Exception {
- com.healthmarketscience.common.util.resource.Handler.init();
- BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
- RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url");
- Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals(batch.size(), queue.size(), "Expected batch size to be the same as amount of queued datums");
- task.PREVIOUSLY_SEEN.put("fake url", batch);
- //Test that it will out previously seen articles
- queue.clear();
- batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals(batch.size(), queue.size(), "Expected batch size to be the same as amount of queued datums");
- }
-
- /**
- * Test that perpetual streams will not output previously seen articles.
- * @throws Exception Exception
- */
- @Test
- public void testPerpetualNoTimeFramePull() throws Exception {
- com.healthmarketscience.common.util.resource.Handler.init();
- BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
- RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", new DateTime().minusYears(10), 10000, true);
- Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals(batch.size(), queue.size(), "Expected batch size to be the same as amount of queued datums");
- task.PREVIOUSLY_SEEN.put("fake url", batch);
- //Test that it will not out previously seen articles
- queue.clear();
- batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals( queue.size(), 0 );
- assertEquals( batch.size(), 20 );
- task.PREVIOUSLY_SEEN.put("fake url", batch);
- //Test that not seen urls aren't blocked.
- queue.clear();
- batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist2.xml"));
- assertEquals(batch.size(), queue.size());
- assertEquals( queue.size(), 25);
- assertEquals( batch.size(), 25);
- }
-
- /**
- * Test that you can task will only output aritcles after a certain published time.
- * @throws Exception Exception
- */
- @Test
- public void testNonPerpetualTimeFramedPull() throws Exception {
- com.healthmarketscience.common.util.resource.Handler.init();
- BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
- DateTime publishedSince = new DateTime().withYear(2014).withDayOfMonth(5).withMonthOfYear(9).withZone(DateTimeZone.UTC);
- RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", publishedSince, 10000, false);
- Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals( queue.size(), 15);
- assertEquals( batch.size(), 20);
- assertTrue( queue.size() < batch.size());
- task.PREVIOUSLY_SEEN.put("fake url", batch);
- //Test that it will out previously seen articles
- queue.clear();
- batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals( queue.size(), 15);
- assertEquals( batch.size(), 20);
- assertTrue( queue.size() < batch.size());
- }
-
- /**
- * Test that task will only output articles after a certain published time that it has not seen before.
- * @throws Exception Exception
- */
- @Test
- public void testPerpetualTimeFramedPull() throws Exception {
- com.healthmarketscience.common.util.resource.Handler.init();
- BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
- DateTime publishedSince = new DateTime().withYear(2014).withDayOfMonth(5).withMonthOfYear(9).withZone(DateTimeZone.UTC);
- RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", publishedSince, 10000, true);
- Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals( queue.size(),15);
- assertEquals( batch.size(), 20);
- assertTrue( queue.size() < batch.size());
- task.PREVIOUSLY_SEEN.put("fake url", batch);
- //Test that it will not out put previously seen articles
- queue.clear();
- batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml"));
- assertEquals( queue.size(), 0);
- assertEquals( batch.size(), 20);
- assertTrue( queue.size() < batch.size());
- task.PREVIOUSLY_SEEN.put("fake url", batch);
-
- batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist2.xml"));
- assertTrue( queue.size() < batch.size());
- assertEquals(queue.size(), 3);
- assertEquals(batch.size(), 25);
- }
-
-
-
-
-}
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTest.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTest.java
deleted file mode 100644
index c3f45ad..0000000
--- a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTest.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.rss.RssStreamConfiguration;
-import org.apache.streams.rss.provider.perpetual.RssFeedScheduler;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.junit.Assert;
-
-/**
- * Unit tests for {@link org.apache.streams.rss.provider.RssStreamProvider}
- */
-public class RssStreamProviderTest extends RandomizedTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RssStreamProviderTest.class);
-
- @Test
- public void testRssFeedShutdownsNonPerpetual() throws Exception {
- RssStreamProvider provider = null;
- try {
- final CountDownLatch latch = new CountDownLatch(1);
- BlockingQueue<StreamsDatum> datums = new LinkedBlockingQueue<>();
- provider = new RssStreamProvider(new RssStreamConfiguration()) {
- @Override
- protected RssFeedScheduler getScheduler(BlockingQueue<StreamsDatum> queue) {
- return new MockScheduler(latch, queue);
- }
- };
- provider.prepare(null);
- int datumCount = 0;
- provider.startStream();
- while (!provider.scheduler.isComplete()) {
- StreamsResultSet batch = provider.readCurrent();
- LOGGER.debug("Batch size : {}", batch.size());
- datumCount += batch.size();
- Thread.sleep(randomIntBetween(0, 3000));
- }
- latch.await();
-
- //one last pull incase of race condition
- StreamsResultSet batch = provider.readCurrent();
- LOGGER.debug("Batch size : {}", batch.size());
- datumCount += batch.size();
- if (batch.size() != 0) {
- //if race condition happened, pull again
- batch = provider.readCurrent();
- Assert.assertEquals(0, batch.size());
- }
-
- Assert.assertTrue(provider.scheduler.isComplete());
- Assert.assertEquals(20, datumCount);
- Assert.assertFalse(provider.isRunning());
- Assert.assertEquals(0, datums.size());
- provider.cleanUp();
- } finally {
- if (provider != null) {
- provider.cleanUp();
- }
- }
- }
-
-
- private class MockScheduler extends RssFeedScheduler {
-
- private BlockingQueue<StreamsDatum> queue;
- private CountDownLatch latch;
- private volatile boolean complete = false;
-
- public MockScheduler(CountDownLatch latch, BlockingQueue<StreamsDatum> dataQueue) {
- super(null, null, dataQueue);
- this.latch = latch;
- this.queue = dataQueue;
- }
-
- @Override
- public void run() {
- try {
- for (int i = 0; i < 20; ++i) {
- this.queue.put(new StreamsDatum(null));
- Thread.sleep(randomIntBetween(0, 5000));
- }
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- } finally {
- this.complete = true;
- this.latch.countDown();
- }
- }
-
-
- @Override
- public boolean isComplete() {
- return this.complete;
- }
- }
-}
-
-
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/perpetual/RssFeedSchedulerTest.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/perpetual/RssFeedSchedulerTest.java
deleted file mode 100644
index bf4720a..0000000
--- a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/perpetual/RssFeedSchedulerTest.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.provider.perpetual;
-
-import org.apache.streams.rss.FeedDetails;
-import org.apache.streams.rss.provider.RssStreamProviderTask;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-
-/**
- * Unit tests for {@link org.apache.streams.rss.provider.perpetual.RssFeedScheduler}
- */
-public class RssFeedSchedulerTest {
-
-
- /**
- * Test that feeds are scheduled based on elapsed time correctly.
- * Takes 1 minute to run.
- */
- @Test
- public void testScheduleFeeds() {
- ExecutorService mockService = mock(ExecutorService.class);
- final List<String> queuedTasks = new ArrayList<>(5);
- doAnswer(invocationOnMock -> {
- queuedTasks.add(((RssStreamProviderTask) invocationOnMock.getArguments()[0]).getRssFeed());
- return null;
- }).when(mockService).execute(any(Runnable.class));
-
- RssFeedScheduler scheduler = new RssFeedScheduler(mockService, createFeedList(), new LinkedBlockingQueue<>(), 1);
- scheduler.scheduleFeeds();
- Assert.assertEquals("Expected 2 Feeds to be scheduled", 2, queuedTasks.size());
- Assert.assertEquals("Expected Feed 1 to be queued first", "1", queuedTasks.get(0));
- Assert.assertEquals("Expected Feed 2 to be queued second", "2", queuedTasks.get(1));
-
- safeSleep(1);
- scheduler.scheduleFeeds();
- Assert.assertEquals("Only feed 1 should have been re-queued", 3, queuedTasks.size());
- Assert.assertEquals("Only feed 1 should have been re-queued", "1", queuedTasks.get(2));
-
- safeSleep(60 * 1000);
- scheduler.scheduleFeeds();
- Assert.assertEquals("Both feeds should have been re-queued", 5, queuedTasks.size());
- Assert.assertEquals("1", queuedTasks.get(3));
- Assert.assertEquals("2", queuedTasks.get(4));
- }
-
- private List<FeedDetails> createFeedList() {
- List<FeedDetails> list = new LinkedList<>();
- FeedDetails fd = new FeedDetails();
- fd.setPollIntervalMillis(1L);
- fd.setUrl("1");
- list.add(fd);
-
- fd = new FeedDetails();
- fd.setPollIntervalMillis(60L * 1000);
- fd.setUrl("2");
- list.add(fd);
- return list;
- }
-
- private void safeSleep(long milliseconds) {
- try {
- Thread.sleep(milliseconds);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
-
-}
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/RssStreamProviderIT.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/RssStreamProviderIT.java
deleted file mode 100644
index c9ded9d..0000000
--- a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/RssStreamProviderIT.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.test;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.rss.FeedDetails;
-import org.apache.streams.rss.RssStreamConfiguration;
-import org.apache.streams.rss.provider.RssStreamProvider;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.LineNumberReader;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.hamcrest.number.OrderingComparison.greaterThan;
-
-public class RssStreamProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RssStreamProviderIT.class);
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Test
- public void testRssStreamProvider() throws Exception {
-
- final String configfile = "./target/test-classes/RssStreamProviderIT.conf";
- final String outfile = "./target/test-classes/RssStreamProviderIT.stdout.txt";
-
- InputStream is = RssStreamProviderIT.class.getResourceAsStream("/top100.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- RssStreamConfiguration configuration = new RssStreamConfiguration();
- List<FeedDetails> feedArray = new ArrayList<>();
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- feedArray.add(new FeedDetails().withUrl(line).withPollIntervalMillis(5000L));
- }
- }
- configuration.setFeeds(feedArray);
- } catch ( Exception ex ) {
- ex.printStackTrace();
- Assert.fail();
- }
-
- org.junit.Assert.assertThat(configuration.getFeeds().size(), greaterThan(70));
-
- OutputStream os = new FileOutputStream(configfile);
- OutputStreamWriter osw = new OutputStreamWriter(os);
- BufferedWriter bw = new BufferedWriter(osw);
-
- // write conf
- ObjectNode feedsNode = mapper.convertValue(configuration, ObjectNode.class);
- JsonNode configNode = mapper.createObjectNode().set("rss", feedsNode);
-
- bw.write(mapper.writeValueAsString(configNode));
- bw.flush();
- bw.close();
-
- File config = new File(configfile);
- assert (config.exists());
- assert (config.canRead());
- assert (config.isFile());
-
- RssStreamProvider.main(new String[]{configfile, outfile});
-
- File out = new File(outfile);
- assert (out.exists());
- assert (out.canRead());
- assert (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- assert (outCounter.getLineNumber() >= 200);
-
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/RssTypeConverterTest.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/RssTypeConverterTest.java
deleted file mode 100644
index 37833c5..0000000
--- a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/RssTypeConverterTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.test;
-
-import org.apache.streams.rss.processor.RssTypeConverter;
-
-import org.apache.commons.lang3.SerializationUtils;
-import org.junit.Test;
-
-/**
- * Tests Serializability of {@link org.apache.streams.rss.processor.RssTypeConverter}
- */
-public class RssTypeConverterTest {
- @Test
- public void testSerializability() {
- RssTypeConverter converter = new RssTypeConverter();
- RssTypeConverter clone = SerializationUtils.clone(converter);
- }
-}
diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/SyndEntryActivitySerializerIT.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/SyndEntryActivitySerializerIT.java
deleted file mode 100644
index 37ff7cf..0000000
--- a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/test/SyndEntryActivitySerializerIT.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.rss.test;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Author;
-import org.apache.streams.pojo.json.Provider;
-import org.apache.streams.rss.serializer.SyndEntryActivitySerializer;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URL;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Objects;
-import java.util.Scanner;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests ability to convert SyndEntry ObjectNode form to {@link org.apache.streams.rss.processor.RssTypeConverter} form
- */
-public class SyndEntryActivitySerializerIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SyndEntryActivitySerializerIT.class);
-
- private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Test
- public void testJsonData() throws Exception {
- Scanner scanner = new Scanner(this.getClass().getResourceAsStream("/TestSyndEntryJson.txt"));
- List<Activity> activities = new LinkedList<>();
- List<ObjectNode> objects = new LinkedList<>();
-
- SyndEntryActivitySerializer serializer = new SyndEntryActivitySerializer();
-
- while (scanner.hasNext()) {
- String line = scanner.nextLine();
- LOGGER.debug(line);
- ObjectNode node = (ObjectNode) mapper.readTree(line);
-
- objects.add(node);
- activities.add(serializer.deserialize(node));
- }
-
- assertEquals(11, activities.size());
-
- for (int x = 0; x < activities.size(); x++) {
- ObjectNode objectNode = objects.get(x);
- Activity activity = activities.get(x);
-
- testActor(objectNode.get("author").asText(), activity.getActor());
- testAuthor(objectNode.get("author").asText(), activity.getObject().getAuthor());
- testProvider("id:providers:rss", "RSS", activity.getProvider());
- validateProviderUrl(activity.getProvider());
- testVerb("post", activity.getVerb());
- testPublished(objectNode.get("publishedDate").asText(), activity.getPublished());
- testUrl(objectNode.get("uri").asText(), objectNode.get("link").asText(), activity);
- }
- }
-
- public void testVerb(String expected, String verb) {
- assertEquals(expected, verb);
- }
-
- public void testPublished(String expected, DateTime published) {
- assertEquals(new DateTime(expected, DateTimeZone.UTC), published);
- }
-
- public void testActor(String expected, ActivityObject actor) {
- assertEquals("id:rss:null" + ":" + expected, actor.getId());
- assertEquals(expected, actor.getDisplayName());
- }
-
- public void testAuthor(String expected, Author author) {
- assertEquals(expected, author.getDisplayName());
- assertEquals(expected, author.getId());
- }
-
- public void testProvider(String expectedId, String expectedDisplay, Provider provider) {
- assertEquals(expectedId, provider.getId());
- assertEquals(expectedDisplay, provider.getDisplayName());
- }
-
- /**
- * validate Provider Url.
- * @param provider Provider
- */
- public void validateProviderUrl(Provider provider) {
- URL url = null;
-
- try {
- url = new URL(provider.getUrl());
- url.toURI();
- } catch (Exception ex) {
- LOGGER.error("Threw an exception while trying to validate URL: {} - {}", provider.getUrl(), ex);
- }
-
- assertNotNull(url);
- }
-
- public void testUrl(String expectedUri, String expectedLink, Activity activity) {
- assertTrue((Objects.equals(expectedUri, activity.getUrl()) || Objects.equals(expectedLink, activity.getUrl())));
- assertTrue((Objects.equals(expectedUri, activity.getObject().getUrl()) ||
- Objects.equals(expectedLink, activity.getObject().getUrl())));
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-rss/src/test/resources/rss.conf b/streams-contrib/streams-provider-rss/src/test/resources/rss.conf
deleted file mode 100644
index ad0946e..0000000
--- a/streams-contrib/streams-provider-rss/src/test/resources/rss.conf
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-feeds = [
- {
- url = "http://feeds.feedburner.com/Techcrunch"
- pollIntervalMillis = 300000
- },
- {
- url = "http://rss.slashdot.org/Slashdot/slashdot"
- pollIntervalMillis = 240000
-
- },
- {
- url = "http://www.engadget.com/rss.xml"
- pollIntervalMillis = 60000
- }
-]
diff --git a/streams-contrib/streams-provider-rss/src/test/resources/top100.txt b/streams-contrib/streams-provider-rss/src/test/resources/top100.txt
deleted file mode 100644
index d5086aa..0000000
--- a/streams-contrib/streams-provider-rss/src/test/resources/top100.txt
+++ /dev/null
@@ -1,100 +0,0 @@
-http://feeds.feedburner.com/Techcrunch
-http://rss.slashdot.org/Slashdot/slashdot
-http://www.engadget.com/rss.xml
-http://googleblog.blogspot.com/atom.xml
-http://scobleizer.wordpress.com/feed/
-http://www.joelonsoftware.com/rss.xml
-http://digg.com/rss/index.xml
-http://feeds.feedburner.com/boingboing/iBag
-http://www.scripting.com/rss.xml
-http://www.lifehacker.com/index.xml
-http://www.techmeme.com/index.xml
-http://feeds.feedburner.com/43Folders
-http://www.alistapart.com/rss.xml
-http://www.wired.com/news_drop/netcenter/netcenter.rdf
-http://www.bloglines.com/rss/about/news
-http://www.readwriteweb.com/rss.xml
-http://feeds.feedburner.com/OmMalik
-http://daringfireball.net/index.xml
-http://radar.oreilly.com/atom.xml
-http://www.techdirt.com/techdirt_rss.xml
-http://feeds.feedburner.com/mashable
-http://feeds.feedburner.com/MicroPersuasion
-http://feeds.feedburner.com/ScottHanselman
-http://www.blogmaverick.com/rss.xml
-http://blog.outer-court.com/rss.xml
-http://feeds.feedburner.com/37signals/beMH
-http://battellemedia.com/index.xml
-http://jeremy.zawodny.com/blog/rss2.xml
-http://weblog.infoworld.com/udell/rss.xml
-http://headrush.typepad.com/creating_passionate_users/index.rdf
-http://www.tapestrycomics.com/dilbert.xml
-http://www.downloadsquad.com/rss.xml
-http://www.tuaw.com/rss.xml
-http://news.com.com/2547-1_3-0-5.xml
-http://google.blogspace.com/index.xml
-http://rssnewsapps.ziffdavis.com/msw.xml
-http://www.gizmodo.net/index.xml
-http://minimsft.blogspot.com/atom.xml
-http://www.zeldman.com/feed/zeldman.xml
-http://feeds.feedburner.com/Crunchnotes
-http://www.mezzoblue.com/rss/index.xml
-http://www.simplebits.com/xml/rss.xml
-http://feeds.feedburner.com/SolutionWatch
-http://www.makezine.com/blog/index.xml
-http://www.kuro5hin.org/backend.rdf
-http://feeds.feedburner.com/ProgrammableWeb
-http://weblogs.asp.net/scottgu/rss.aspx
-http://www.hackaday.com/rss.xml
-http://TheDailyWTF.com/rss.aspx
-http://feeds.feedburner.com/tapestrydilbert
-http://www.gapingvoid.com/index.rdf
-http://www.codinghorror.com/blog/index.xml
-http://partners.userland.com/people/docSearls.xml
-http://dig.csail.mit.edu/breadcrumbs/blog/feed/4
-http://arstechnica.com/index.ars/rss
-http://www.macrumors.com/macrumors.xml
-http://www.calacanis.com/rss.xml
-http://37signals.com/svn/index_full.rdf
-http://rssnewsapps.ziffdavis.com/extreme.xml
-http://headrush.typepad.com/creating_passionate_users/atom.xml
-http://www.kottke.org/index.xml
-http://feeds.feedburner.com/vitaminmasterfeed
-http://news.com.com/2547-1_3-0-20.xml
-http://www.thinksecret.com/rss.xml
-http://www.betanews.com/rss2
-http://www.ysearchblog.com/index.xml
-http://feeds.kottke.org/main
-http://www.mattcutts.com/blog/feed/
-http://msdn.microsoft.com/rss.xml
-http://feeds.feedburner.com/ventureblog
-http://feeds.feedburner.com/eHub
-http://googlereader.blogspot.com/atom.xml
-http://feeds.feedburner.com/Mobilecrunch
-http://sethgodin.typepad.com/seths_blog/atom.xml
-http://www.kk.org/cooltools/index.xml
-http://feeds.gawker.com/lifehacker/full
-http://feeds.feedburner.com/Flickrblog
-http://www.pvrblog.com/pvr/index.rdf
-http://pluralsight.com/blogs/dbox/Rss.aspx
-http://joi.ito.com/index.xml
-http://feeds.feedburner.com/BurnThisRSS2
-http://software.ericsink.com/rss.xml
-http://radar.oreilly.com/index.rdf
-http://blogs.msdn.com/ie/Rss.aspx
-http://feeds.feedburner.com/letTheGoodTimesRollByGuyKawasaki
-http://www.larkware.com/larkware2.xml
-http://www.456bereastreet.com/feed.xml
-http://www.pbs.org/cringely/pulpit/rss2.xml
-http://ranchero.com/xml/rss.xml
-http://www.niallkennedy.com/blog/index.xml
-http://socialsoftware.weblogsinc.com/rss.xml
-http://sethgodin.typepad.com/seths_blog/index.rdf
-http://www.wired.com/news/feeds/rss2/0,2610,,00.xml
-http://www.nytimes.com/services/xml/rss/nyt/Technology.xml
-http://www.alistapart.com/feed/rss.xml
-http://chris.pirillo.com/index.xml
-http://www.dpreview.com/news/dpr.rdf
-http://www.lessig.org/blog/index.rdf
-http://feeds.yuiblog.com/YahooUserInterfaceBlog
-http://rss.cnn.com/rss/cnn_topstories.rss
diff --git a/streams-contrib/streams-provider-sysomos/README.md b/streams-contrib/streams-provider-sysomos/README.md
deleted file mode 100644
index 6d0d227..0000000
--- a/streams-contrib/streams-provider-sysomos/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-provider-sysomos
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-provider-sysomos/pom.xml b/streams-contrib/streams-provider-sysomos/pom.xml
deleted file mode 100644
index d9d5fbf..0000000
--- a/streams-contrib/streams-provider-sysomos/pom.xml
+++ /dev/null
@@ -1,189 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-provider-sysomos</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Sysomos Provider</description>
-
- <properties>
- <skipITs>true</skipITs>
- <testDataBaseURl>http://streams.peoplepattern.com.s3.amazonaws.com/test-data/</testDataBaseURl>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml</groupId>
- <artifactId>aalto-xml</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>com.sysomos</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.jvnet.jaxb2.maven2</groupId>
- <artifactId>maven-jaxb2-plugin</artifactId>
- <configuration>
- <schemaDirectory>src/main/xmlschema/com/sysomos/</schemaDirectory>
- <generateDirectory>target/generated-sources/jaxb2</generateDirectory>
- <verbose>true</verbose>
- <debug>true</debug>
- <encoding>${project.build.sourceEncoding}</encoding>
- <forceRegenerate>true</forceRegenerate>
- <removeOldOutput>false</removeOldOutput>
- <generatePackage>com.sysomos.xml</generatePackage>
- <plugins>
- <plugin>
- <groupId>org.jvnet.jaxb2_commons</groupId>
- <artifactId>jaxb2-basics</artifactId>
- <version>${jaxb2-basics.version}</version>
- </plugin>
- </plugins>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- <source>target/generated-sources/jaxb2</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>download-it-data</id>
- <phase>pre-integration-test</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
- <url>${testDataBaseURl}/${project.artifactId}.zip</url>
- <unpack>true</unpack>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- <!--<md5>df65b5642f33676313ebe4d5b69a3fff</md5>-->
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <configuration>
- <skipTests>${skipITs}</skipTests>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </plugin>
- </plugins>
-
- </build>
-</project>
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/SysomosException.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/SysomosException.java
deleted file mode 100644
index 1480308..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/SysomosException.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos;
-
-/**
- * Runtime exception wrapper for Sysomos Errors.
- */
-public class SysomosException extends RuntimeException {
-
- private int errorCode = -1;
-
- public SysomosException() {
- // TODO Auto-generated constructor stub
- }
-
- public SysomosException(String arg0) {
- super(arg0);
- // TODO Auto-generated constructor stub
- }
-
- public SysomosException(Throwable arg0) {
- super(arg0);
- // TODO Auto-generated constructor stub
- }
-
- public SysomosException(String arg0, Throwable arg1) {
- super(arg0, arg1);
- // TODO Auto-generated constructor stub
- }
-
- public SysomosException(String arg0, int errorCode) {
- super(arg0);
- this.errorCode = errorCode;
- }
-
- public SysomosException(String arg0, Throwable arg1, int errorCode) {
- super(arg0, arg1);
- this.errorCode = errorCode;
- }
-
- public int getErrorCode() {
- return this.errorCode;
- }
-
-
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/conversion/SysomosBeatActivityConverter.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/conversion/SysomosBeatActivityConverter.java
deleted file mode 100644
index c0278cd..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/conversion/SysomosBeatActivityConverter.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.conversion;
-
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Provider;
-
-import com.sysomos.xml.BeatApi;
-import org.apache.commons.lang.StringUtils;
-import org.joda.time.DateTime;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.streams.data.util.ActivityUtil.LANGUAGE_EXTENSION;
-import static org.apache.streams.data.util.ActivityUtil.LOCATION_EXTENSION;
-import static org.apache.streams.data.util.ActivityUtil.LOCATION_EXTENSION_COUNTRY;
-import static org.apache.streams.data.util.ActivityUtil.getObjectId;
-import static org.apache.streams.data.util.ActivityUtil.getPersonId;
-import static org.apache.streams.data.util.ActivityUtil.getProviderId;
-
-/**
- * Converts an instance of a {@link com.sysomos.xml.BeatApi.BeatResponse.Beat} to an {@link org.apache.streams.pojo.json.Activity}
- */
-public class SysomosBeatActivityConverter {
-
- private static final String LANGUAGE_KEY = "LANGUAGE";
-
- /**
- * convert BeatApi.BeatResponse.Beat to Activity
- * @param beat BeatApi.BeatResponse.Beat
- * @return Activity
- */
- public Activity convert(BeatApi.BeatResponse.Beat beat) {
- Activity converted = new Activity();
- converted.setId(beat.getDocid());
- converted.setVerb("post");
- converted.setContent(beat.getContent());
- converted.setTitle(beat.getTitle());
- converted.setPublished(new DateTime(beat.getTime()));
- converted.setUrl(beat.getLink());
- converted.setActor(new ActivityObject());
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(converted);
- extensions.put("keywords", beat.getContent());
- setLocation(beat, extensions);
- setObject(beat, converted);
- setProvider(beat, converted);
- Map<String, BeatApi.BeatResponse.Beat.Tag> mappedTags = mapTags(beat);
- setLanguage(mappedTags, extensions);
- extensions.put("sysomos", beat);
-
- setChannelSpecificValues(beat, converted, mappedTags);
-
- return converted;
- }
-
- protected void setChannelSpecificValues(
- BeatApi.BeatResponse.Beat beat,
- Activity converted, Map<String, BeatApi.BeatResponse.Beat.Tag> mappedTags) {
- String mediaType = beat.getMediaType();
- String lowerMediaType = mediaType.toLowerCase();
- ActivityObject actor = converted.getActor();
- ActivityObject object = converted.getObject();
- if ("TWITTER".equals(mediaType)) {
- actor.setId(getPersonId(lowerMediaType, beat.getHost()));
- actor.setDisplayName(beat.getHost());
- actor.setUrl("http://twitter.com/" + beat.getHost());
- object.setObjectType("tweet");
- object.setId(getObjectId(lowerMediaType, "tweet", beat.getTweetid()));
- } else if ("FACEBOOK".equals(mediaType)) {
- String fbid = mappedTags.containsKey("FBID") ? mappedTags.get("FBID").getValue() : "";
- actor.setId(getPersonId(lowerMediaType, fbid));
- actor.setDisplayName(beat.getTitle());
- actor.setUrl(beat.getHost());
- object.setObjectType("post");
- object.setId(getObjectId(lowerMediaType, "post", String.valueOf(converted.getContent().hashCode())));
- } else {
- actor.setId(null);
- actor.setDisplayName(null);
- actor.setUrl(null);
- object.setObjectType("post");
- object.setId(getObjectId(lowerMediaType, "post", String.valueOf(converted.getContent().hashCode())));
- }
- }
-
- protected void setLanguage(Map<String, BeatApi.BeatResponse.Beat.Tag> mappedTags, Map<String, Object> extensions) {
- if (mappedTags.containsKey(LANGUAGE_KEY)) {
- extensions.put(LANGUAGE_EXTENSION, mappedTags.get(LANGUAGE_KEY).getValue());
- }
- }
-
- protected void setObject(BeatApi.BeatResponse.Beat beat, Activity converted) {
- ActivityObject object = new ActivityObject();
- converted.setObject(object);
- object.setUrl(beat.getLink());
- object.setContent(beat.getContent());
- }
-
- @SuppressWarnings("unchecked")
- protected void setLocation(BeatApi.BeatResponse.Beat beat, Map<String, Object> extensions) {
- Map<String, Object> location;
- String country = beat.getLocation().getCountry();
- if (StringUtils.isNotBlank(country)) {
- if (extensions.containsKey(LOCATION_EXTENSION)) {
- location = (Map<String, Object>) extensions.get(LOCATION_EXTENSION);
- } else {
- location = new HashMap<>();
- extensions.put(LOCATION_EXTENSION, location);
- }
- location.put(LOCATION_EXTENSION_COUNTRY, country);
- }
- }
-
- protected void setProvider(BeatApi.BeatResponse.Beat beat, Activity converted) {
- Provider provider = new Provider();
- String mediaType = beat.getMediaType().toLowerCase();
- provider.setId(getProviderId(mediaType));
- provider.setDisplayName(StringUtils.capitalize(mediaType));
- converted.setProvider(provider);
- }
-
- protected Map<String, BeatApi.BeatResponse.Beat.Tag> mapTags(BeatApi.BeatResponse.Beat beat) {
- Map<String, BeatApi.BeatResponse.Beat.Tag> tags = new HashMap<>();
- for (BeatApi.BeatResponse.Beat.Tag tag : beat.getTag()) {
- if (tag.getSystemType() != null) {
- tags.put(tag.getSystemType().trim(), tag);
- }
- }
- return tags;
- }
-
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/data/HeartbeatInfo.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/data/HeartbeatInfo.java
deleted file mode 100644
index eede9f4..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/data/HeartbeatInfo.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.data;
-
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
-
-import java.io.ByteArrayInputStream;
-import java.util.ArrayList;
-import java.util.List;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
-/**
- * Represents Heatbeat metadata from the Sysomos API.
- */
-public class HeartbeatInfo {
-
- private Document doc;
- private List<SysomosTagDefinition> tags;
-
- /**
- * HeartbeatInfo constructor.
- * @param xmlString xmlString
- * @throws Exception Exception
- */
- public HeartbeatInfo(String xmlString) throws Exception {
- DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
- DocumentBuilder docBuilder = dbFactory.newDocumentBuilder();
- this.doc = docBuilder.parse(new InputSource(new ByteArrayInputStream(xmlString.getBytes("utf-8"))));
- this.tags = new ArrayList<SysomosTagDefinition>();
- createTagDefinitions();
- }
-
- private void createTagDefinitions() {
- this.tags = new ArrayList<SysomosTagDefinition>();
- NodeList tagList = this.doc.getElementsByTagName("tag");
-
- for (int i = 0; i < tagList.getLength(); ++i) {
- Node tag = tagList.item(i);
- SysomosTagDefinition tagDefinition = createSysomosTagDefinitionFromNode(tag);
- if (this.hasTagName(tagDefinition.getTagName())) {
- SysomosTagDefinition otherTag = this.getTagWithTagName(tagDefinition.getTagName());
- if (!otherTag.getDisplayName().equals(tagDefinition.getDisplayName())) {
- throw new RuntimeException("A single tag ("
- + otherTag.getTagName()
- + ") has multiple display names ("
- + otherTag.getDisplayName()
- + " , "
- + tagDefinition.getDisplayName()
- + ")");
- } else {
- List<String> otherQueries = otherTag.getQueries();
- for (String query : tagDefinition.getQueries()) {
- if (!otherQueries.contains(query)) {
- otherTag.addQuery(query);
- }
- }
- }
- } else {
- this.tags.add(tagDefinition);
- }
-
- }
- }
-
- private SysomosTagDefinition createSysomosTagDefinitionFromNode(Node tag) {
- Element tagElement = (Element) tag;
- SysomosTagDefinition tagDefinition = new SysomosTagDefinition(tagElement.getElementsByTagName("name").item(0).getTextContent(),
- tagElement.getElementsByTagName("displayName").item(0).getTextContent());
- NodeList taggingRule = tagElement.getElementsByTagName("taggingRule");
- for (int i = 0; i < taggingRule.getLength(); ++i) {
- Element rule = (Element) taggingRule.item(i);
- NodeList queries = rule.getElementsByTagName("query");
- for (int j = 0; j < queries.getLength(); ++j) {
- Element query = (Element) queries.item(j);
- tagDefinition.addQuery(query.getTextContent());
- }
- }
- return tagDefinition;
- }
-
- /**
- * hasTagName.
- * @param tagName tagName
- * @return hasTagName
- */
- public boolean hasTagName(String tagName) {
- for (SysomosTagDefinition tag : this.tags) {
- if (tag.hasTagName(tagName)) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * getTagWithTagName.
- * @param tagName tagName
- * @return SysomosTagDefinition
- */
- public SysomosTagDefinition getTagWithTagName(String tagName) {
- for (SysomosTagDefinition tag : this.tags) {
- if (tag.hasTagName(tagName)) {
- return tag;
- }
- }
- return null;
- }
-
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/data/SysomosTagDefinition.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/data/SysomosTagDefinition.java
deleted file mode 100644
index 889db31..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/data/SysomosTagDefinition.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.data;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Represents a Sysomos Heartbeat Tag. These tags are defined within the Sysomos system and exposed via the data feed.
- */
-public class SysomosTagDefinition {
-
- private String tagName;
- private String displayName;
- private List<String> queries;
-
- /**
- * SysomosTagDefinition constructor.
- * @param tagName tagName
- * @param displayName displayName
- */
- public SysomosTagDefinition(String tagName, String displayName) {
- this.tagName = tagName;
- this.displayName = displayName;
- this.queries = new ArrayList<String>();
- }
-
- public String getTagName() {
- return this.tagName;
- }
-
- public String getDisplayName() {
- return this.displayName;
- }
-
- /**
- * getQueries.
- * @return Queries
- */
- public List<String> getQueries() {
- List<String> result = new ArrayList<String>();
- result.addAll(this.queries);
- return result;
- }
-
- public void addQuery(String query) {
- this.queries.add(query);
- }
-
- public boolean hasTagName(String tagName) {
- return this.tagName.equals(tagName);
- }
-
- public boolean hasQuery(String query) {
- return this.queries.contains(query);
- }
-
- public boolean hasDisplayName(String displayName) {
- return this.displayName.equals(displayName);
- }
-
- @Override
- public boolean equals(Object object) {
- if (!(object instanceof SysomosTagDefinition)) {
- return false;
- }
- SysomosTagDefinition that = (SysomosTagDefinition) object;
- if (!this.tagName.equals(that.tagName)) {
- return false;
- }
- if (!this.displayName.equals(that.displayName)) {
- return false;
- }
- if (this.queries.size() != that.queries.size()) {
- return false;
- }
- for (int i = 0; i < this.queries.size(); ++i) {
- if (!that.queries.contains(this.queries.get(i))) {
- return false;
- }
- }
- return true;
- }
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/processor/SysomosTypeConverter.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/processor/SysomosTypeConverter.java
deleted file mode 100644
index 3f228b4..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/processor/SysomosTypeConverter.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.sysomos.conversion.SysomosBeatActivityConverter;
-
-import com.sysomos.xml.BeatApi;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Stream processor that converts Sysomos type to Activity.
- */
-public class SysomosTypeConverter implements StreamsProcessor {
-
- public static final String STREAMS_ID = "SysomosTypeConverter";
-
- private SysomosBeatActivityConverter converter;
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- if (entry.getDocument() instanceof BeatApi.BeatResponse.Beat) {
- entry.setDocument(converter.convert((BeatApi.BeatResponse.Beat)entry.getDocument()));
- return Stream.of(entry).collect(Collectors.toList());
- } else {
- return new ArrayList<>();
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
- converter = new SysomosBeatActivityConverter();
- }
-
- @Override
- public void cleanUp() {
- //NOP
- }
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/AbstractRequestBuilder.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/AbstractRequestBuilder.java
deleted file mode 100644
index de604b4..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/AbstractRequestBuilder.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.provider;
-
-import org.apache.streams.sysomos.SysomosException;
-import org.apache.streams.sysomos.util.SysomosUtils;
-
-import com.sysomos.xml.BeatApi;
-import com.sysomos.xml.ObjectFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.StringReader;
-import java.net.URL;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Unmarshaller;
-
-/**
- * Defines a common pattern for requesting data from the Sysomos API.
- */
-public abstract class AbstractRequestBuilder implements RequestBuilder {
- private static final Logger LOGGER = LoggerFactory.getLogger(AbstractRequestBuilder.class);
-
- /**
- * Executes the request to the Sysomos Heartbeat API and returns a valid response.
- */
- public BeatApi.BeatResponse execute() {
- URL url = this.getRequestUrl();
- try {
- String xmlResponse = SysomosUtils.queryUrl(url);
- JAXBContext context = JAXBContext.newInstance(ObjectFactory.class);
- Unmarshaller unmarshaller = context.createUnmarshaller();
- BeatApi beatApi = (BeatApi) unmarshaller.unmarshal(new StringReader(xmlResponse));
- return beatApi.getBeatResponse();
- } catch (JAXBException ex) {
- LOGGER.error("Unable to unmarshal XML content");
- throw new SysomosException("Unable to unmarshal XML content", ex);
- }
- }
-
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/ContentRequestBuilder.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/ContentRequestBuilder.java
deleted file mode 100644
index 7ae47cf..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/ContentRequestBuilder.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.provider;
-
-import org.apache.streams.sysomos.SysomosException;
-
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.MalformedURLException;
-import java.net.URL;
-
-import static org.apache.streams.sysomos.util.SysomosUtils.SYSOMOS_DATE_FORMATTER;
-
-/**
- * Builds requests for the Sysomos Heartbeat Content API. This is the preferred method of
- * accessing data from Sysomoos Heartbeat
- */
-public class ContentRequestBuilder extends AbstractRequestBuilder implements RequestBuilder {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ContentRequestBuilder.class);
-
- private String baseUrl;
- private String hid;
- private String addedAfter;
- private String addedBefore;
- private String size;
- private String offset;
- private String apiKey;
-
- /**
- * The max number of items you are allowed to get per request.
- */
- public static final int MAX_ALLOWED_PER_REQUEST = 10000;
-
- /**
- * Constructs a new ContentRequestBuilder for the specified API key and Sysomos URL.
- * @param baseUrl the base URL for the sysomos API
- * @param apiKey the API key generated by Sysomos for authorization
- */
- protected ContentRequestBuilder(String baseUrl, String apiKey) {
- this.baseUrl = baseUrl;
- this.apiKey = apiKey;
- }
-
- /**
- * Gets the Request URL based on the local fields.
- * @return a valid URL for the Sysomos API or an exception
- */
- @Override
- public URL getRequestUrl() {
- StringBuilder url = new StringBuilder();
- url.append(this.baseUrl);
- url.append("v1/heartbeat/content?");
- url.append("apiKey=");
- url.append(this.apiKey);
- url.append("&hid=");
- url.append(this.hid);
- if (size != null) {
- url.append("&size=");
- url.append(this.size);
- }
- if (this.addedAfter != null) {
- url.append("&addedAfter=");
- url.append(this.addedAfter);
- }
- if (this.addedBefore != null) {
- url.append("&addedBefore=");
- url.append(this.addedBefore);
- }
- if (this.offset != null) {
- url.append("&offset=");
- url.append(this.offset);
- }
- String urlString = url.toString();
- LOGGER.debug("Constructed Request URL: {}", urlString);
- try {
- return new URL(urlString);
- } catch (MalformedURLException ex) {
- throw new SysomosException("Malformed Request URL. Check Request Builder parameters", ex);
- }
- }
-
- @Override
- public RequestBuilder setHeartBeatId(int hid) {
- return setHeartBeatId(Integer.toString(hid));
- }
-
- @Override
- public RequestBuilder setHeartBeatId(String hid) {
- this.hid = hid;
- return this;
- }
-
- @Override
- public RequestBuilder setAddedAfterDate(DateTime afterDate) {
- this.addedAfter = SYSOMOS_DATE_FORMATTER.print(afterDate);
- return this;
- }
-
- @Override
- public RequestBuilder setAddedBeforeDate(DateTime beforeDate) {
- this.addedBefore = SYSOMOS_DATE_FORMATTER.print(beforeDate);
- return this;
- }
-
- @Override
- public RequestBuilder setReturnSetSize(long size) {
- this.size = Long.toString(Math.min(size, MAX_ALLOWED_PER_REQUEST));
- return this;
- }
-
- @Override
- public RequestBuilder setOffset(long offset) {
- this.offset = Long.toString(offset);
- return this;
- }
-
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/RequestBuilder.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/RequestBuilder.java
deleted file mode 100644
index 53887af..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/RequestBuilder.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.provider;
-
-import com.sysomos.xml.BeatApi;
-import org.joda.time.DateTime;
-
-import java.net.URL;
-
-/**
- * Simplifying abstraction that aids in building a request to the Sysomos API in a chained fashion.
- */
-public interface RequestBuilder {
-
- /**
- * Sets the date after which documents should be returned from Sysomos.
- * @param afterDate the {@link org.joda.time.DateTime} instance representing the after date
- *
- * @return The RequestBuilder for continued Chaining
- */
- RequestBuilder setAddedAfterDate(DateTime afterDate);
-
- /**
- * Sets the date before which documents should be returned from Sysomos.
- * @param beforeDate the {@link org.joda.time.DateTime} instance representing the before date
- *
- * @return The RequestBuilder for continued Chaining
- */
- RequestBuilder setAddedBeforeDate(DateTime beforeDate);
-
- /**
- * Sets the size of the expected response.
- * @param size the number of documents
- *
- * @return The RequestBuilder for continued Chaining
- */
- RequestBuilder setReturnSetSize(long size);
-
- /**
- * Sets the starting offset for the number of documents given the other parameters.
- * @param offset the starting offset
- *
- * @return The RequestBuilder for continued Chaining
- */
- RequestBuilder setOffset(long offset);
-
- /**
- * Sets the Sysomos Heartbeat ID.
- * @param hid Heartbeat ID
- *
- * @return The RequestBuilder for continued Chaining
- */
- RequestBuilder setHeartBeatId(int hid);
-
- /**
- * Sets the Sysomos Heartbeat ID as a String.
- * @param hid Heartbeat ID string
- *
- * @return The RequestBuilder for continued Chaining
- */
- RequestBuilder setHeartBeatId(String hid);
-
- /**
- * Returns the full url need to execute a request.
- *
- * <p/>
- * Example:
- * http://api.sysomos.com/dev/v1/heartbeat/content?apiKey=YOUR
- * -APIKEY&hid=YOUR-HEARTBEAT-ID&offset=0&size=10&
- * addedAfter=2010-10-15T13:00:00Z&addedBefore=2010-10-18T13:00:00Z
- *
- * @return the URL to use when making requests of Sysomos Heartbeat
- */
- URL getRequestUrl();
-
- /**
- * Executes the request to the Sysomos Heartbeat API and returns a valid response
- */
- BeatApi.BeatResponse execute();
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosClient.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosClient.java
deleted file mode 100644
index 95d8ad9..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosClient.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.provider;
-
-import java.net.HttpURLConnection;
-
-/**
- * Wrapper for the Sysomos API.
- */
-public class SysomosClient {
-
- public static final String BASE_URL_STRING = "http://api.sysomos.com/";
- private static final String HEARTBEAT_INFO_URL = "http://api.sysomos.com/v1/heartbeat/info?apiKey={api_key}&hid={hid}";
-
- private String apiKey;
-
- private HttpURLConnection client;
-
- public SysomosClient(String apiKey) {
- this.apiKey = apiKey;
- }
-
- public RequestBuilder createRequestBuilder() {
- return new ContentRequestBuilder(BASE_URL_STRING, this.apiKey);
- }
-
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java
deleted file mode 100644
index 71447cb..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.provider;
-
-import org.apache.streams.core.StreamsDatum;
-
-import com.sysomos.xml.BeatApi;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Provides a {@link java.lang.Runnable} query mechanism for grabbing documents from the Sysomos API
- */
-public class SysomosHeartbeatStream implements Runnable {
-
- private enum OperatingMode { DATE, DOC_MATCH }
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SysomosHeartbeatStream.class);
-
- private final SysomosProvider provider;
- private final SysomosClient client;
- private final String heartbeatId;
- private final long maxApiBatch;
- private final long minLatency;
- private final OperatingMode mode;
-
- private String lastId;
- private DateTime beforeTime;
- private DateTime afterTime;
- private DateTime lastRunTime;
- private int offsetCount = 0;
- private boolean enabled = true;
-
- public SysomosHeartbeatStream(SysomosProvider provider, String heartbeatId) {
- this(provider, heartbeatId, null, DateTime.now());
- }
-
- /**
- * SysomosHeartbeatStream constructor.
- * @param provider SysomosProvider
- * @param heartbeatId heartbeatId
- * @param beforeTime DateTime
- * @param afterTime DateTime
- */
- public SysomosHeartbeatStream(SysomosProvider provider, String heartbeatId, DateTime beforeTime, DateTime afterTime) {
- this(provider, heartbeatId, OperatingMode.DATE);
- this.beforeTime = beforeTime;
- this.afterTime = afterTime;
- }
-
- /**
- * SysomosHeartbeatStream constructor.
- * @param provider SysomosProvider
- * @param heartbeatId heartbeatId
- * @param documentId last documentId
- */
- public SysomosHeartbeatStream(SysomosProvider provider, String heartbeatId, String documentId) {
- this(provider, heartbeatId, OperatingMode.DOC_MATCH);
- this.lastId = documentId;
- }
-
- /**
- * SysomosHeartbeatStream constructor.
- * @param provider SysomosProvider
- * @param heartbeatId heartbeatId
- * @param mode OperatingMode
- */
- public SysomosHeartbeatStream(SysomosProvider provider, String heartbeatId, OperatingMode mode) {
- this.provider = provider;
- this.heartbeatId = heartbeatId;
-
- this.client = provider.getClient();
- this.maxApiBatch = provider.getMaxApiBatch();
- this.minLatency = provider.getMinLatency();
- this.mode = mode;
- }
-
- @Override
- public void run() {
- try {
- executeRun();
- } catch (Exception ex) {
- LOGGER.error("Error executing heartbeat stream", ex);
- shutdown();
- }
- }
-
- protected void executeRun() {
- QueryResult result;
- String mostCurrentId = null;
- int totalDocCount = 0;
- lastRunTime = DateTime.now();
- //Iff we are trying to get to a specific document ID, continue to query after minimum delay
- do {
- LOGGER.debug("Querying API to match last ID of {} or time range of {} - {}", lastId, afterTime, beforeTime);
- result = queryApi();
- totalDocCount += result.getResponseSize();
- //Ensure that we are only assigning lastId to the latest ID, even if there is backfill query.
- //Since offset is calcuated at the end of the run, if we detect the need to backfill, it will increment to 1
- if (offsetCount == 1) {
- mostCurrentId = result.getCurrentId();
- }
- updateOffset(result);
- }
- while (offsetCount > 0);
-
- updateState(result, mostCurrentId, totalDocCount);
- LOGGER.debug("Completed current execution with a final docID of {} or time of {}", lastId, afterTime);
- }
-
- protected void updateState(QueryResult result, String mostCurrentId, int totalDocCount) {
- if (OperatingMode.DOC_MATCH.equals(mode)) {
- //Set the last ID so that the next time we are executed we will continue to query only so long as we haven't
- //found the specific ID
- lastId = mostCurrentId == null ? result.getCurrentId() : mostCurrentId;
- } else {
- //If we didn't see any docs, there might be a lag on the Sysomos side. Retry.
- afterTime = totalDocCount == 0 ? afterTime : lastRunTime;
- }
-
- if (SysomosProvider.Mode.BACKFILL_AND_TERMINATE.equals(provider.getMode())) {
- shutdown();
- LOGGER.info("Completed backfill to {} for heartbeat {}", OperatingMode.DOC_MATCH.equals(mode) ? lastId : afterTime, heartbeatId);
- }
- }
-
- protected void updateOffset(QueryResult result) {
- if (OperatingMode.DOC_MATCH.equals(mode)) {
- //Reset the offset iff we have found a match or this is the first execution
- offsetCount = lastId == null || result.isMatchedLastId() ? 0 : offsetCount + 1;
- } else {
- offsetCount = result.getResponseSize() == 0 ? 0 : offsetCount + 1;
- }
- if (offsetCount > 0) {
- sleep();
- }
- }
-
- protected void sleep() {
- try {
- Thread.sleep(this.minLatency);
- } catch (InterruptedException ex) {
- LOGGER.warn("Thread interrupted while sleeping minimum delay", ex);
- shutdown();
- }
- }
-
- protected QueryResult queryApi() {
- BeatApi.BeatResponse response = executeApiRequest();
-
- String currentId = null;
- boolean matched = false;
- int responseSize = 0;
- if (response != null) {
- for (BeatApi.BeatResponse.Beat beat : response.getBeat()) {
- String docId = beat.getDocid();
- //We get documents in descending time order. This will set the id to the latest document
- if (currentId == null) {
- currentId = docId;
- }
- //We only want to process documents that we know we have not seen before
- if (lastId != null && lastId.equals(docId)) {
- matched = true;
- break;
- }
- StreamsDatum item = new StreamsDatum(beat, docId);
- item.getMetadata().put("heartbeat", this.heartbeatId);
- this.provider.enqueueItem(item);
- }
- responseSize = response.getCount();
- }
- return new QueryResult(matched, currentId, responseSize);
- }
-
- protected BeatApi.BeatResponse executeApiRequest() {
- BeatApi.BeatResponse response = null;
- try {
- if (enabled) {
- RequestBuilder requestBuilder = this.client.createRequestBuilder()
- .setHeartBeatId(heartbeatId)
- .setOffset(offsetCount * maxApiBatch)
- .setReturnSetSize(maxApiBatch);
- if (beforeTime != null) {
- requestBuilder.setAddedBeforeDate(beforeTime);
- }
- if (afterTime != null) {
- requestBuilder.setAddedAfterDate(afterTime);
- }
- response = requestBuilder.execute();
-
- LOGGER.debug("Received {} results from API query", response.getCount());
- }
- } catch (Exception ex) {
- LOGGER.warn("Error querying Sysomos API", ex);
- }
- return response;
- }
-
- protected void shutdown() {
- provider.signalComplete(heartbeatId);
- enabled = false;
- }
-
- protected class QueryResult {
- private boolean matchedLastId;
- private String currentId;
- private int responseSize;
-
-
- public QueryResult(boolean matchedLastId, String currentId, int responseSize) {
- this.matchedLastId = matchedLastId;
- this.currentId = currentId;
- this.responseSize = responseSize;
- }
-
- public boolean isMatchedLastId() {
- return matchedLastId;
- }
-
- public void setMatchedLastId(boolean matchedLastId) {
- this.matchedLastId = matchedLastId;
- }
-
- public String getCurrentId() {
- return currentId;
- }
-
- public void setCurrentId(String currentId) {
- this.currentId = currentId;
- }
-
- public int getResponseSize() {
- return responseSize;
- }
-
- public void setResponseSize(int responseSize) {
- this.responseSize = responseSize;
- }
- }
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosProvider.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosProvider.java
deleted file mode 100644
index a77a129..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosProvider.java
+++ /dev/null
@@ -1,383 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.data.util.RFC3339Utils;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.sysomos.SysomosConfiguration;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.math.BigInteger;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * Streams Provider for the Sysomos Heartbeat API
- * <p/>
- * Configuration:
- * The provider takes either a Map[String,Object] containing the mode (backfill and terminate OR continuous) and a
- * Map[String,String] of heartbeat IDs to document target ids or a string of the format
- * ${heartbeatId}:${documentId},...,${heartbeatId}:${documentId}
- * This configuration will configure the provider to backfill to the specified document and either terminate or not
- * depending on the mode flag. Continuous mode is assumed, and is the ony mode supported by the String configuration.
- */
-public class SysomosProvider implements StreamsProvider {
-
- public static final String STREAMS_ID = "SysomosProvider";
- public static final String ENDING_TIME_KEY = "addedBefore";
- public static final String STARTING_TIME_KEY = "addedAfter";
- public static final String MODE_KEY = "mode";
- public static final String STARTING_DOCS_KEY = "startingDocs";
- public static final int LATENCY = 10000; //Default minLatency for querying the Sysomos API in milliseconds
- public static final long PROVIDER_BATCH_SIZE = 10000L; //Default maximum size of the queue
- public static final long API_BATCH_SIZE = 1000L; //Default maximum size of an API request
- private static final Logger LOGGER = LoggerFactory.getLogger(SysomosProvider.class);
- private final ReadWriteLock lock = new ReentrantReadWriteLock();
- private final Set<String> completedHeartbeats = new HashSet<>();
- private final long maxQueued;
- private final long minLatency;
- private final long scheduledLatency;
- private final long maxApiBatch;
- protected volatile Queue<StreamsDatum> providerQueue;
- private SysomosClient client;
- private SysomosConfiguration config;
- private ScheduledExecutorService stream;
- private Map<String, String> documentIds;
- private Map<String, String> addedBefore;
- private Map<String, String> addedAfter;
- private Mode mode = Mode.CONTINUOUS;
- private boolean started = false;
- private AtomicInteger count;
- /**
- * SysomosProvider constructor.
- * @param sysomosConfiguration SysomosConfiguration
- */
- public SysomosProvider(SysomosConfiguration sysomosConfiguration) {
- this.config = sysomosConfiguration;
- this.client = new SysomosClient(sysomosConfiguration.getApiKey());
- this.maxQueued = sysomosConfiguration.getMaxBatchSize() == null ? PROVIDER_BATCH_SIZE : sysomosConfiguration.getMaxBatchSize();
- this.minLatency = sysomosConfiguration.getMinDelayMs() == null ? LATENCY : sysomosConfiguration.getMinDelayMs();
- this.scheduledLatency = sysomosConfiguration.getScheduledDelayMs() == null
- ? (LATENCY * 15) : sysomosConfiguration.getScheduledDelayMs();
- this.maxApiBatch = sysomosConfiguration.getMinDelayMs() == null ? API_BATCH_SIZE : sysomosConfiguration.getApiBatchSize();
- this.count = new AtomicInteger();
- }
-
- /**
- * To use from command line:
- * <p/>
- * Supply configuration similar to src/test/resources/rss.conf
- * <p/>
- * Launch using:
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.rss.provider.RssStreamProvider -Dexec.args="rss.conf articles.json"
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- SysomosConfiguration config = new ComponentConfigurator<>(SysomosConfiguration.class).detectConfiguration(typesafe, "rss");
- SysomosProvider provider = new SysomosProvider(config);
-
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- try {
- json = mapper.writeValueAsString(datum.getDocument());
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while (provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-
- public SysomosConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(SysomosConfiguration config) {
- this.config = config;
- }
-
- public Mode getMode() {
- return mode;
- }
-
- public long getMinLatency() {
- return minLatency;
- }
-
- public long getMaxApiBatch() {
- return maxApiBatch;
- }
-
- public SysomosClient getClient() {
- return client;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
- LOGGER.trace("Starting Producer");
- if (!started) {
- LOGGER.trace("Producer not started. Initializing");
- stream = Executors.newScheduledThreadPool(getConfig().getHeartbeatIds().size() + 1);
- for (String heartbeatId : getConfig().getHeartbeatIds()) {
- Runnable task = createStream(heartbeatId);
- stream.scheduleWithFixedDelay(task, 0, this.scheduledLatency, TimeUnit.MILLISECONDS);
- LOGGER.info("Started producer task for heartbeat {}", heartbeatId);
- }
- started = true;
- }
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- StreamsResultSet current;
- try {
- lock.writeLock().lock();
- LOGGER.debug("Creating new result set for {} items", providerQueue.size());
- count.addAndGet(providerQueue.size());
- current = new StreamsResultSet(providerQueue);
- providerQueue = constructQueue();
- } finally {
- lock.writeLock().unlock();
- }
-
- return current;
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger bigInteger) {
- throw new NotImplementedException("readNew not currently implemented");
- }
-
- @Override
- public StreamsResultSet readRange(DateTime dateTime, DateTime dateTime2) {
- throw new NotImplementedException("readRange not currently implemented");
- }
-
- //If the provider queue still has data, we are still running. If not, we are running if we have not been signaled
- //by all completed heartbeats so long as the thread pool is alive
- @Override
- public boolean isRunning() {
- return providerQueue.size() > 0
- || (completedHeartbeats.size() < this.getConfig().getHeartbeatIds().size()
- && !(stream.isTerminated()
- || stream.isShutdown()));
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.providerQueue = constructQueue();
- if (configurationObject instanceof Map) {
- extractConfigFromMap((Map)configurationObject);
- } else if (configurationObject instanceof String) {
- documentIds = Splitter.on(";").trimResults().withKeyValueSeparator("=").split((String)configurationObject);
- }
- }
-
- @Override
- public void cleanUp() {
- stream.shutdown(); // Disable new tasks from being submitted
- try {
- // Wait a while for existing tasks to terminate
- if (!stream.awaitTermination(60, TimeUnit.SECONDS)) {
- stream.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!stream.awaitTermination(60, TimeUnit.SECONDS)) {
- LOGGER.error("Stream did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- stream.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * signalComplete.
- * @param heartbeatId heartbeatId
- */
- public void signalComplete(String heartbeatId) {
- try {
- this.lock.writeLock().lock();
- this.completedHeartbeats.add(heartbeatId);
- if (!this.isRunning()) {
- this.cleanUp();
- }
- } finally {
- this.lock.writeLock().unlock();
- }
-
- }
-
- protected void enqueueItem(StreamsDatum datum) {
- boolean success;
- do {
- try {
- pauseForSpace(); //Dont lock before this pause. We don't want to block the readCurrent method
- lock.readLock().lock();
- success = providerQueue.offer(datum);
- Thread.yield();
- } finally {
- lock.readLock().unlock();
- }
- }
- while (!success);
- }
-
- protected SysomosHeartbeatStream createStream(String heartbeatId) {
- String afterTime = addedAfter != null && addedAfter.containsKey(heartbeatId) ? addedAfter.get(heartbeatId) : null;
- String beforeTime = addedBefore != null && addedBefore.containsKey(heartbeatId) ? addedBefore.get(heartbeatId) : null;
-
- if (documentIds != null && documentIds.containsKey(heartbeatId)) {
- return new SysomosHeartbeatStream(this, heartbeatId, documentIds.get(heartbeatId));
- }
- if (afterTime != null) {
- if (beforeTime != null) {
- return new SysomosHeartbeatStream(this, heartbeatId, RFC3339Utils.parseToUTC(beforeTime), RFC3339Utils.parseToUTC(afterTime));
- } else {
- return new SysomosHeartbeatStream(this, heartbeatId, null, RFC3339Utils.parseToUTC(afterTime));
- }
- }
- return new SysomosHeartbeatStream(this, heartbeatId);
- }
-
- /**
- * Wait for the queue size to be below threshold before allowing execution to continue on this thread.
- */
- protected void pauseForSpace() {
- while (this.providerQueue.size() >= maxQueued) {
- LOGGER.trace("Sleeping the current thread due to a full queue");
- try {
- Thread.sleep(100);
- LOGGER.trace("Resuming thread after wait period");
- } catch (InterruptedException ex) {
- LOGGER.warn("Thread was interrupted", ex);
- }
- }
- }
-
- @SuppressWarnings("unchecked")
- protected void extractConfigFromMap(Map configMap) {
- if (configMap.containsKey(MODE_KEY)) {
- Object configMode = configMap.get(MODE_KEY);
- if (!(configMode instanceof Mode)) {
- throw new IllegalStateException("Invalid configuration. Mode must be an instance of the Mode enum but was " + configMode);
- }
- this.mode = (Mode)configMode;
- }
- if (configMap.containsKey(STARTING_DOCS_KEY)) {
- Object configIds = configMap.get(STARTING_DOCS_KEY);
- if (!(configIds instanceof Map)) {
- throw new IllegalStateException("Invalid configuration. StartingDocs must be an instance of Map<String,String> but was "
- + configIds);
- }
- this.documentIds = (Map)configIds;
- }
- if (configMap.containsKey(STARTING_TIME_KEY)) {
- Object configIds = configMap.get(STARTING_TIME_KEY);
- if (!(configIds instanceof Map)) {
- throw new IllegalStateException("Invalid configuration. Added after key must be an instance of Map<String,String> but was "
- + configIds);
- }
- this.addedAfter = (Map)configIds;
- }
- if (configMap.containsKey(ENDING_TIME_KEY)) {
- Object configIds = configMap.get(ENDING_TIME_KEY);
- if (!(configIds instanceof Map)) {
- throw new IllegalStateException("Invalid configuration. Added before key must be an instance of Map<String,String> but was "
- + configIds);
- }
- this.addedBefore = (Map)configIds;
- }
- }
-
- private Queue<StreamsDatum> constructQueue() {
- return new ConcurrentLinkedQueue<>();
- }
-
- public int getCount() {
- return this.count.get();
- }
-
- public enum Mode {CONTINUOUS, BACKFILL_AND_TERMINATE}
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/util/SysomosUtils.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/util/SysomosUtils.java
deleted file mode 100644
index 0acc81f..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/util/SysomosUtils.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.util;
-
-import org.apache.streams.sysomos.SysomosException;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.StringWriter;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Provides utilities for working with Sysomos.
- */
-public class SysomosUtils {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SysomosUtils.class);
-
- public static final Pattern CODE_PATTERN = Pattern.compile("code: ([0-9]+)");
- public static final DateTimeFormatter SYSOMOS_DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss'Z'").withZoneUTC();
-
- private SysomosUtils() {}
-
- /**
- * Queries the sysomos URL and provides the response as a String.
- *
- * @param url the Sysomos URL to query
- * @return valid XML String
- */
- public static String queryUrl(URL url) {
- try {
- HttpURLConnection cn = (HttpURLConnection) url.openConnection();
- cn.setRequestMethod("GET");
- cn.addRequestProperty("Content-Type", "text/xml;charset=UTF-8");
- cn.setDoInput(true);
- cn.setDoOutput(false);
- StringWriter writer = new StringWriter();
- IOUtils.copy(new InputStreamReader(cn.getInputStream()), writer);
- writer.flush();
-
- String xmlResponse = writer.toString();
- if (StringUtils.isEmpty(xmlResponse)) {
- throw new SysomosException("XML Response from Sysomos was empty : "
- + xmlResponse
- + "\n"
- + cn.getResponseMessage(),
- cn.getResponseCode());
- }
- return xmlResponse;
- } catch (IOException ex) {
- LOGGER.error("Error executing request : {}", ex, url.toString());
- String message = ex.getMessage();
- Matcher match = CODE_PATTERN.matcher(message);
- if (match.find()) {
- int errorCode = Integer.parseInt(match.group(1));
- throw new SysomosException(message, ex, errorCode);
- } else {
- throw new SysomosException(ex.getMessage(), ex);
- }
- }
- }
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/Sysomos.json b/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/Sysomos.json
deleted file mode 100644
index b5feb61..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/Sysomos.json
+++ /dev/null
@@ -1,61 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType": "org.apache.streams.sysomos.Sysomos",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "tweetJsonLink": {
- "type": "string"
- },
- "location": {
- "properties": {
- "locationString": {
- "type": "string"
- },
- "country": {
- "type": "string"
- }
- }
- },
- "link": {
- "type": "string"
- },
- "twitterFollowing": {
- "type": "string"
- },
- "twitterFollowers": {
- "type": "string"
- },
- "tweetid": {
- "type": "string"
- },
- "mediaType": {
- "type": "string"
- },
- "content": {
- "type": "string"
- },
- "docid": {
- "type": "string"
- },
- "sentiment": {
- "type": "string"
- },
- "time": {
- "type": "string"
- },
- "title": {
- "type": "string"
- },
- "tweetHbLink": {
- "type": "string"
- },
- "influenceLevel": {
- "type": "string"
- }
- }
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/SysomosConfiguration.json b/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/SysomosConfiguration.json
deleted file mode 100644
index a678f3c..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/jsonschema/org/apache/streams/sysomos/SysomosConfiguration.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.sysomos.SysomosConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "apiKey": {
- "type": "string"
- },
- "endpoint": {
- "type": "string",
- "description": "The endpoint"
- },
- "heartbeatIds": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "string"
- }
- },
- "minDelayMs": {
- "type": "String",
- "format": "utc-millisec"
- },
- "scheduledDelayMs": {
- "type": "String",
- "format": "utc-millisec"
- },
- "maxBatchSize": {
- "type": "String",
- "format": "utc-millisec"
- },
- "apiBatchSize": {
- "type": "String",
- "format": "utc-millisec"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-sysomos/src/main/resources/components.dot b/streams-contrib/streams-provider-sysomos/src/main/resources/components.dot
deleted file mode 100644
index ba87c08..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/resources/components.dot
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- subgraph cluster_generators {
- label="generators";
- style="dotted";
- generators_sysomos [label="https://api.sysomos.com"]
- }
-
- subgraph cluster_providers {
- label="providers";
- providers_sysomos [label="SysomosProvider"]
- }
-
- subgraph cluster_processors {
- label="processors";
- processors_sysomos_typeconveter [label="SysomosTypeConverter"]
- }
-
- persisters [label="persisters", shape="cylinder"];
-
- generators_sysomos -> providers_sysomos
- providers_sysomos -> processors_sysomos_typeconveter [label="c.s.x.BeatApi.BeatResponse.Beat"]
-
- processors_sysomos_typeconveter -> persisters [label="o.a.s.p.j.Activity"]
-
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/main/resources/reference.conf b/streams-contrib/streams-provider-sysomos/src/main/resources/reference.conf
deleted file mode 100644
index f754b78..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/resources/reference.conf
+++ /dev/null
@@ -1,5 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-sysomos {
- endpoint = "http://api.sysomos.com/"
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-sysomos/src/main/xmlschema/com/sysomos/binding.xjb b/streams-contrib/streams-provider-sysomos/src/main/xmlschema/com/sysomos/binding.xjb
deleted file mode 100644
index d94a76b..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/xmlschema/com/sysomos/binding.xjb
+++ /dev/null
@@ -1,33 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<jaxb:bindings xmlns:jaxb="http://java.sun.com/xml/ns/jaxb" xmlns:xs="http://www.w3.org/2001/XMLSchema"
- xmlns:xjc="http://java.sun.com/xml/ns/jaxb/xjc" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://java.sun.com/xml/ns/jaxb http://java.sun.com/xml/ns/jaxb/bindingschema_2_0.xsd"
- version="2.1">
- <jaxb:globalBindings>
- <!-- Use java.util.Calendar instead of javax.xml.datatype.XMLGregorianCalendar for xs:dateTime -->
- <jaxb:javaType name="java.util.Calendar" xmlType="xs:dateTime"
- parseMethod="javax.xml.bind.DatatypeConverter.parseDateTime"
- printMethod="javax.xml.bind.DatatypeConverter.printDateTime" />
-
- <!-- Force all classes implements Serializable -->
- <xjc:serializable uid="1" />
- </jaxb:globalBindings>
-
-</jaxb:bindings>
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-sysomos/src/main/xmlschema/com/sysomos/sysomos.xsd b/streams-contrib/streams-provider-sysomos/src/main/xmlschema/com/sysomos/sysomos.xsd
deleted file mode 100644
index d5b949b..0000000
--- a/streams-contrib/streams-provider-sysomos/src/main/xmlschema/com/sysomos/sysomos.xsd
+++ /dev/null
@@ -1,91 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<xs:schema attributeFormDefault="unqualified" elementFormDefault="qualified" targetNamespace="http://www.blogscope.net/api/namespace/schema/SearchResponse" xmlns:xs="http://www.w3.org/2001/XMLSchema">
- <xs:element name="beatApi">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="beatRequest">
- <xs:complexType>
- <xs:sequence>
- <xs:element type="xs:string" name="heatbeatID"/>
- <xs:element type="xs:dateTime" name="startDate"/>
- <xs:element type="xs:dateTime" name="endDate"/>
- <xs:element type="xs:string" name="key"/>
- <xs:element type="xs:short" name="size"/>
- <xs:element type="xs:byte" name="offset"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="beatResponse">
- <xs:complexType>
- <xs:sequence>
- <xs:element name="beat" maxOccurs="unbounded" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element type="xs:string" name="docid"/>
- <xs:element type="xs:string" name="mediaType"/>
- <xs:element type="xs:dateTime" name="time"/>
- <xs:element type="xs:string" name="title"/>
- <xs:element type="xs:string" name="content"/>
- <xs:element type="xs:string" name="host"/>
- <xs:element type="xs:anyURI" name="link"/>
- <xs:element type="xs:string" name="gender" minOccurs="0"/>
- <xs:element name="location">
- <xs:complexType>
- <xs:sequence>
- <xs:element type="xs:string" name="country"/>
- <xs:element type="xs:string" name="locationString"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element type="xs:string" name="influenceLevel"/>
- <xs:element type="xs:string" name="sentiment"/>
- <xs:element name="tag" maxOccurs="unbounded" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element type="xs:string" name="value"/>
- <xs:element type="xs:string" name="displayName"/>
- <xs:element type="xs:string" name="systemType" minOccurs="0"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element type="xs:string" name="tweetid" minOccurs="0"/>
- <xs:element type="xs:anyURI" name="tweetJsonLink" minOccurs="0"/>
- <xs:element type="xs:anyURI" name="tweetHbLink" minOccurs="0"/>
- <xs:element type="xs:string" name="twitterFollowers" minOccurs="0"/>
- <xs:element type="xs:string" name="twitterFollowing" minOccurs="0"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element type="xs:short" name="count"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- <xs:element name="errors" maxOccurs="1" minOccurs="0">
- <xs:complexType>
- <xs:sequence>
- <xs:element type="xs:boolean" name="error"/>
- <xs:element type="xs:string" name="errorMessage"/>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
- </xs:sequence>
- </xs:complexType>
- </xs:element>
-</xs:schema>
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-sysomos/src/site/markdown/index.md b/streams-contrib/streams-provider-sysomos/src/site/markdown/index.md
deleted file mode 100644
index 3be1de3..0000000
--- a/streams-contrib/streams-provider-sysomos/src/site/markdown/index.md
+++ /dev/null
@@ -1,25 +0,0 @@
-streams-provider-sysomos
-==============
-
-streams-provider-sysomos contains schema definitions, providers, conversions, and utility classes.
-
-## Data Types
-
-| Schema |
-|--------|
-| [Sysomos.json](com/sysomos/Sysomos.json "Sysomos.json") [Sysomos.json.html](apidocs/com/sysomos/Sysomos.html "javadoc") |
-| [Sysomos.xsd](com/sysomos/sysomos.xsd "Sysomos.xsd") [Sysomos.xsd.html](apidocs/com/sysomos/BeatApi.html "javadoc") |
-
-## Configuration
-
-| Schema |
-|--------|
-| [SysomosConfiguration.json](com/sysomos/SysomosConfiguration.json "SysomosConfiguration.json") [SysomosConfiguration.html](apidocs/com/sysomos/SysomosConfiguration.html "javadoc") |
-
-## Components
-
-
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosJsonSerDeIT.java b/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosJsonSerDeIT.java
deleted file mode 100644
index 56555ad..0000000
--- a/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosJsonSerDeIT.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.test;
-
-import org.apache.streams.sysomos.Sysomos;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests ability to convert String json form to {@link org.apache.streams.sysomos.Sysomos} form
- */
-public class SysomosJsonSerDeIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SysomosJsonSerDeIT.class);
-
- private ObjectMapper mapper = new ObjectMapper();
-
- @Test
- public void testSysomosJsonSerDe() {
-
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- InputStream is = SysomosJsonSerDeIT.class.getResourceAsStream("/sysomos_jsons.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- LOGGER.debug(line);
-
- Sysomos ser = mapper.readValue(line, Sysomos.class);
-
- String des = mapper.writeValueAsString(ser);
- LOGGER.debug(des);
- }
- } catch (Exception ex) {
- ex.printStackTrace();
- Assert.fail();
- }
- }
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java b/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java
deleted file mode 100644
index 091552f..0000000
--- a/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.test;
-
-import com.fasterxml.aalto.stax.InputFactoryImpl;
-import com.fasterxml.aalto.stax.OutputFactoryImpl;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule;
-import com.fasterxml.jackson.dataformat.xml.XmlFactory;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import com.sysomos.xml.BeatApi;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * Tests ability to convert String xml form to {@link com.sysomos.xml.BeatApi} form
- */
-public class SysomosXmlSerDeIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SysomosXmlSerDeIT.class);
-
- private XmlMapper xmlMapper;
-
- /**
- * before.
- */
- @Before
- public void before() {
-
- XmlFactory xmlFactory = new XmlFactory(new InputFactoryImpl(),
- new OutputFactoryImpl());
-
- JacksonXmlModule module = new JacksonXmlModule();
-
- module.setDefaultUseWrapper(false);
-
- xmlMapper = new XmlMapper(xmlFactory, module);
-
- xmlMapper
- .configure(
- DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY,
- Boolean.TRUE);
- xmlMapper
- .configure(
- DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT,
- Boolean.TRUE);
- xmlMapper
- .configure(
- DeserializationFeature.USE_JAVA_ARRAY_FOR_JSON_ARRAY,
- Boolean.TRUE);
- xmlMapper.configure(
- DeserializationFeature.READ_ENUMS_USING_TO_STRING,
- Boolean.TRUE);
-
- }
-
- @Test
- public void test() {
-
- InputStream is = SysomosXmlSerDeIT.class.getResourceAsStream("/sysomos_xmls.txt");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- LOGGER.debug(line);
-
- BeatApi ser = xmlMapper.readValue(line, BeatApi.class);
-
- String des = xmlMapper.writeValueAsString(ser);
- LOGGER.debug(des);
- }
- } catch ( Exception ex ) {
- ex.printStackTrace();
- Assert.fail();
- }
- }
-}
diff --git a/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/provider/SysomosProviderIT.java b/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/provider/SysomosProviderIT.java
deleted file mode 100644
index 7d0a7d3..0000000
--- a/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/provider/SysomosProviderIT.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.sysomos.test.provider;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.sysomos.provider.SysomosProvider;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-/**
- * Integration test for SysomosProviderIT.
- *
- * @Ignore this is ignored because the project doesn't have credentials to test it with during CI
- */
-public class SysomosProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SysomosProviderIT.class);
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Test(enabled = false)
- public void testSysomosProvider() throws Exception {
-
- String configfile = "./target/test-classes/SysomosProviderIT.conf";
- String outfile = "./target/test-classes/SysomosProviderIT.stdout.txt";
-
- SysomosProvider.main(new String[]{configfile, outfile});
-
- File out = new File(outfile);
- assert (out.exists());
- assert (out.canRead());
- assert (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- assert (outCounter.getLineNumber() >= 1);
-
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-sysomos/src/test/resources/sysomos.conf b/streams-contrib/streams-provider-sysomos/src/test/resources/sysomos.conf
deleted file mode 100644
index 13af8b8..0000000
--- a/streams-contrib/streams-provider-sysomos/src/test/resources/sysomos.conf
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-sysomos {
- heartbeatIds = [
- HBID
- ]
- apiBatchSize = 500
- apiKey = KEY
- minDelayMs = 10000
- scheduledDelayMs = 120000
- maxBatchSize = 10000
-}
diff --git a/streams-contrib/streams-provider-twitter/README.md b/streams-contrib/streams-provider-twitter/README.md
deleted file mode 100644
index faab613..0000000
--- a/streams-contrib/streams-provider-twitter/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-provider-twitter
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-provider-twitter/pom.xml b/streams-contrib/streams-provider-twitter/pom.xml
deleted file mode 100644
index 304ddde..0000000
--- a/streams-contrib/streams-provider-twitter/pom.xml
+++ /dev/null
@@ -1,222 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-provider-twitter</artifactId>
-
- <properties>
- <skipITs>true</skipITs>
- <testDataBaseURl>http://streams.peoplepattern.com.s3.amazonaws.com/test-data/</testDataBaseURl>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>com.typesafe</groupId>
- <artifactId>config</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-joda</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-converters</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-http</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path-assert</artifactId>
- </dependency>
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>hbc-core</artifactId>
- <version>2.2.0</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.juneau</groupId>
- <artifactId>juneau-rest-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-all</artifactId>
- <version>1.3</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.twitter.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <version>1.2.1</version>
- <executions>
- <execution>
- <id>download-it-data</id>
- <phase>pre-integration-test</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
- <url>${testDataBaseURl}/${project.artifactId}.zip</url>
- <unpack>true</unpack>
- <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
- <!--<md5>df65b5642f33676313ebe4d5b69a3fff</md5>-->
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <configuration>
- <skipTests>${skipITs}</skipTests>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>1.5.0</version>
- <executions>
- <execution>
- <goals>
- <goal>exec</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Account.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Account.java
deleted file mode 100644
index dfe0c4e..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Account.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.User;
-
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-
-import java.util.List;
-
-/**
- * Interface for /account methods.
- */
-@Remoteable(path = "https://api.twitter.com/1.1/account")
-public interface Account {
-
- /**
- * Returns settings (including current trend, geo and sleep time information) for the authenticating user.
- *
- * @return AccountSettingsResponse
- * @see <a href=https://dev.twitter.com/rest/reference/get/account/settings">https://dev.twitter.com/rest/reference/get/account/settings</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/settings.json")
- public AccountSettings settings();
-
- /**
- * Returns user credentials for the authenticating user.
- *
- * @return User
- * @see <a href=https://dev.twitter.com/rest/reference/get/account/verify_credentials">https://dev.twitter.com/rest/reference/get/account/verify_credentials</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/verify_credentials.json")
- public User verifyCredentials();
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/AccountActivity.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/AccountActivity.java
deleted file mode 100644
index fe13db0..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/AccountActivity.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.User;
-
-import org.apache.juneau.remoteable.Path;
-import org.apache.juneau.remoteable.Query;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-import org.apache.juneau.rest.client.RestCallException;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.List;
-
-/**
- * Interface for /account_activity methods.
- *
- * @see <a href="https://dev.twitter.com/rest/reference">https://dev.twitter.com/rest/reference</a>
- */
-@Remoteable(path = "https://api.twitter.com/1.1/account_activity")
-public interface AccountActivity {
-
- /**
- * Returns all URLs and their statuses for the given app. Currently, only one webhook URL can be registered to an application.
- *
- * We mark a URL as invalid if it fails the daily validation check. In order to re-enable the URL, call the update endpoint.
- *
- * @return List\<Webhook\>
- * @see <a href=https://https://dev.twitter.com/webhooks/reference/get/account_activity/webhooks">https://https://dev.twitter.com/webhooks/reference/get/account_activity/webhooks</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/webhooks.json")
- public List<Webhook> getWebhooks();
-
- /**
- * Registers a new webhook URL for the given application context. The URL will be validated via CRC request before saving. In case the validation fails, a comprehensive error is returned. message to the requester.
- *
- * Only one webhook URL can be registered to an application.
- *
- * @return Webhook
- * @see <a href=https://dev.twitter.com/webhooks/reference/post/account_activity/webhooks">https://dev.twitter.com/webhooks/reference/post/account_activity/webhooks</a>
- *
- */
- @RemoteMethod(httpMethod = "POST", path = "/webhooks.json")
- public Webhook registerWebhook(@Query("url") String url);
-
- /**
- * Removes the webhook from the provided application’s configuration. The webhook ID can be accessed by making a call to GET /1.1/account_activity/webhooks.
- *
- * @return Boolean
- * @see <a href=https://dev.twitter.com/webhooks/reference/del/account_activity/webhooks">https://dev.twitter.com/webhooks/reference/del/account_activity/webhooks</a>
- *
- */
- @RemoteMethod(httpMethod = "DELETE", path = "/webhooks/{webhookId}.json")
- public Boolean deleteWebhook(@Path("webhookId") Long webhookId);
-
- /**
- * Triggers the challenge response check (CRC) for the given webhook’s URL. If the check is successful, returns 204 and reenables the webhook by setting its status to valid.
- *
- * @param webhookId Webhook ID. Defined in resource path.
- * @return Boolean
- * @see <a href=https://dev.twitter.com/webhooks/reference/put/account_activity/webhooks">https://dev.twitter.com/webhooks/reference/put/account_activity/webhooks</a>
- *
- */
- @RemoteMethod(httpMethod = "PUT", path = "/webhooks/{webhookId}.json")
- public Boolean putWebhook(@Path("webhookId") Long webhookId);
-
- /**
- * Provides a way to determine if a webhook configuration is subscribed to the provided user’s Direct Messages. If the provided user context has an active subscription with the provided app, returns 204 OK. If the response code is not 204, then the user does not have an active subscription. See HTTP Response code and error messages below for details.
- *
- * @param webhookId Webhook ID. Defined in resource path.
- * @return Boolean
- * @see <a href=https://dev.twitter.com/webhooks/reference/put/account_activity/webhooks">https://dev.twitter.com/webhooks/reference/put/account_activity/webhooks</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/webhooks/{webhookId}/subscriptions.json")
- public Boolean getWebhookSubscription(@Path("webhookId") Long webhookId)
- throws InvocationTargetException, RestCallException
- ;
-
- /**
- * Subscribes the provided app to events for the provided user context. When subscribed, all DM events for the provided user will be sent to the app’s webhook via POST request.
- *
- * @param webhookId Webhook ID. Defined in resource path.
- * @return Boolean
- * @see <a href=https://dev.twitter.com/webhooks/reference/post/account_activity/webhooks/subscriptions">https://dev.twitter.com/webhooks/reference/post/account_activity/webhooks/subscriptions</a>
- *
- */
- @RemoteMethod(httpMethod = "POST", path = "/webhooks/{webhookId}/subscriptions.json")
- public Boolean registerWebhookSubscriptions(@Path("webhookId") Long webhookId)
- throws InvocationTargetException, RestCallException
- ;
-
- /**
- * Deactivates subscription for the provided user context and app. After deactivation, all DM events for the requesting user will no longer be sent to the webhook URL.
- *
- * @param webhookId Webhook ID. Defined in resource path.
- * @return Boolean
- * @see <a href=https://dev.twitter.com/webhooks/reference/del/account_activity/webhooks/subscriptions">https://dev.twitter.com/webhooks/reference/del/account_activity/webhooks/subscriptions</a>
- *
- */
- @RemoteMethod(httpMethod = "DELETE", path = "/webhooks/{webhookId}/subscriptions.json")
- public Boolean deleteWebhookSubscriptions(@Path("webhookId") Long webhookId)
- throws InvocationTargetException, RestCallException
- ;
-
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/DirectMessages.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/DirectMessages.java
deleted file mode 100644
index 502750b..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/DirectMessages.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.DirectMessageEvent;
-import org.apache.streams.twitter.pojo.User;
-import org.apache.streams.twitter.pojo.WelcomeMessage;
-
-import org.apache.juneau.remoteable.Body;
-import org.apache.juneau.remoteable.Query;
-import org.apache.juneau.remoteable.QueryIfNE;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-import org.apache.juneau.remoteable.RequestBean;
-
-/**
- * Interface for /direct_messages methods.
- *
- * @see <a href="https://dev.twitter.com/rest/reference">https://dev.twitter.com/rest/reference</a>
- */
-@Remoteable(path = "https://api.twitter.com/1.1/direct_messages")
-public interface DirectMessages {
-
- /**
- * Returns all Direct Message events (both sent and received) within the last 30 days. Sorted in reverse-chronological order.
- *
- * @return {@link org.apache.streams.twitter.api.EventsListResponse}
- * @see <a href="https://dev.twitter.com/rest/reference/get/direct_messages/events/list">https://dev.twitter.com/rest/reference/get/direct_messages/events/list</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/events/list.json")
- public EventsListResponse listEvents(@QueryIfNE EventsListRequest parameters);
-
- /**
- * Returns a single Direct Message event by the given id.
- *
- * @return {@link org.apache.streams.twitter.api.EventShowResponse}
- * @see <a href="https://dev.twitter.com/rest/reference/get/direct_messages/events/show">https://dev.twitter.com/rest/reference/get/direct_messages/events/show</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/events/show.json")
- public EventShowResponse showEvent(@Query("id") Long id);
-
- /**
- * Publishes a new message_create event resulting in a Direct Message sent to a specified user from the authenticating user. Returns an event in the requested format if successful.
- *
- * @return {@link org.apache.streams.twitter.pojo.DirectMessageEvent}
- * @see <a href="https://dev.twitter.com/rest/reference/get/direct_messages/events/new">https://dev.twitter.com/rest/reference/get/direct_messages/events/new</a>
- *
- */
- @RemoteMethod(httpMethod = "POST", path = "/events/new.json")
- public DirectMessageEvent newEvent(@Body MessageCreateRequest event);
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Favorites.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Favorites.java
deleted file mode 100644
index b68e88b..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Favorites.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.Tweet;
-
-import org.apache.juneau.remoteable.Body;
-import org.apache.juneau.remoteable.QueryIfNE;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-
-import java.util.List;
-
-/**
- * Interface for /favorites methods.
- *
- * @see <a href="https://dev.twitter.com/rest/reference">https://dev.twitter.com/rest/reference</a>
- */
-@Remoteable(path = "https://api.twitter.com/1.1/favorites")
-public interface Favorites {
-
- /**
- * Returns the 20 most recent Tweets favorited by the authenticating or specified user.
- * Bind this at: /favorites
- *
- * @param parameters {@link org.apache.streams.twitter.api.FavoritesListRequest}
- * @return List < Tweet >
- * @see <a href="https://dev.twitter.com/rest/reference/get/favorites/list">https://dev.twitter.com/rest/reference/get/favorites/list</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/list.json")
- public List<Tweet> list( @QueryIfNE("*") FavoritesListRequest parameters);
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Followers.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Followers.java
deleted file mode 100644
index 8c2698b..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Followers.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.Tweet;
-
-import org.apache.juneau.remoteable.QueryIfNE;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-
-import java.util.List;
-
-/**
- * Interface for /followers methods.
- *
- * @see <a href="https://dev.twitter.com/rest/reference">https://dev.twitter.com/rest/reference</a>
- */
-@Remoteable(path = "https://api.twitter.com/1.1/followers")
-public interface Followers {
-
- /**
- * Returns a cursored collection of user IDs for every user following the specified user.
- *
- * @param parameters {@link org.apache.streams.twitter.api.FollowersIdsRequest}
- * @return FollowersIdsResponse
- * @see <a href="https://dev.twitter.com/rest/reference/get/followers/ids">https://dev.twitter.com/rest/reference/get/followers/ids</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/ids.json")
- public FollowersIdsResponse ids( @QueryIfNE("*") FollowersIdsRequest parameters);
-
- /**
- * Returns a cursored collection of user objects for users following the specified user.
- *
- * @param parameters {@link org.apache.streams.twitter.api.FollowersListRequest}
- * @return FollowersListResponse
- * @see <a href="https://dev.twitter.com/rest/reference/get/followers/list">https://dev.twitter.com/rest/reference/get/followers/list</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/list.json")
- public FollowersListResponse list( @QueryIfNE("*") FollowersListRequest parameters);
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Friends.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Friends.java
deleted file mode 100644
index 7f75603..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Friends.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.Tweet;
-
-import org.apache.juneau.remoteable.QueryIfNE;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-
-import java.util.List;
-
-/**
- * Interface for /friends methods.
- *
- * @see <a href="https://dev.twitter.com/rest/reference">https://dev.twitter.com/rest/reference</a>
- */
-@Remoteable(path = "https://api.twitter.com/1.1/friends")
-public interface Friends {
-
- /**
- * Returns a cursored collection of user IDs for every user the specified user is following.
- *
- * @param parameters {@link org.apache.streams.twitter.api.FriendsIdsRequest}
- * @return FriendsIdsResponse
- * @see <a href="https://dev.twitter.com/rest/reference/get/friends/ids">https://dev.twitter.com/rest/reference/get/friends/ids</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/ids.json")
- public FriendsIdsResponse ids( @QueryIfNE("*") FriendsIdsRequest parameters);
-
- /**
- * Returns a cursored collection of user objects for every user the specified user is following.
- *
- * @param parameters {@link org.apache.streams.twitter.api.FriendsListRequest}
- * @return FriendsListResponse
- * @see <a href="https://dev.twitter.com/rest/reference/get/friends/list">https://dev.twitter.com/rest/reference/get/friends/list</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/list.json")
- public FriendsListResponse list( @QueryIfNE("*") FriendsListRequest parameters);
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Friendships.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Friendships.java
deleted file mode 100644
index 288db98..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Friendships.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.Friendship;
-import org.apache.streams.twitter.pojo.User;
-
-import org.apache.juneau.remoteable.QueryIfNE;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-
-import java.util.List;
-
-/**
- * Interface for /friendships methods.
- *
- * @see <a href="https://dev.twitter.com/rest/reference">https://dev.twitter.com/rest/reference</a>
- */
-@Remoteable(path = "https://api.twitter.com/1.1/friendships")
-public interface Friendships {
-
- /**
- * Allows the authenticating users to follow the user specified in the ID parameter.
- *
- * @param parameters {@link FollowersIdsRequest}
- * @return FollowersIdsResponse
- * @see <a href="https://dev.twitter.com/rest/reference/post/friendships/create">https://dev.twitter.com/rest/reference/post/friendships/create</a>
- *
- */
- @RemoteMethod(httpMethod = "POST", path = "/create.json")
- public User create(@QueryIfNE("*") FriendshipCreateRequest parameters);
-
- /**
- * Allows the authenticating user to unfollow the user specified in the ID parameter.
- *
- * @param parameters {@link User}
- * @return User
- * @see <a href="https://dev.twitter.com/rest/reference/post/friendships/destroy">https://dev.twitter.com/rest/reference/post/friendships/destroy</a>
- *
- */
- @RemoteMethod(httpMethod = "POST", path = "/destroy.json")
- public User destroy(@QueryIfNE("*") FriendshipDestroyRequest parameters);
-
- /**
- * Returns the relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided.
- * Values for connections can be: following, following_requested, followed_by, none, blocking, muting.
- *
- * @param parameters {@link FriendshipsLookupRequest}
- * @return List<Friendship>
- * @see <a href="https://dev.twitter.com/rest/reference/get/friendships/lookup">https://dev.twitter.com/rest/reference/get/friendships/lookup</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/lookup.json")
- public List<Friendship> lookup(@QueryIfNE("*") FriendshipsLookupRequest parameters);
-
- /**
- * Returns a collection of numeric IDs for every protected user for whom the authenticating user has a pending follow request.
- *
- * @param parameters {@link FriendshipsOutgoingRequest}
- * @return FollowersListResponse
- * @see <a href="https://dev.twitter.com/rest/reference/get/friendships/outgoing">https://dev.twitter.com/rest/reference/get/friendships/outgoing</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/outgoing.json")
- public User outgoing(@QueryIfNE("*") FriendshipsOutgoingRequest parameters);
-
- /**
- * Returns detailed information about the relationship between two arbitrary users.
- *
- * @param parameters {@link FriendshipShowRequest}
- * @return FriendshipShowResponse
- * @see <a href="https://dev.twitter.com/rest/reference/get/friendships/show">https://dev.twitter.com/rest/reference/get/friendships/show</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/show.json")
- public FriendshipShowResponse outgoing(@QueryIfNE("*") FriendshipShowRequest parameters);
-
- /**
- * Allows one to enable or disable retweets and device notifications from the specified user.
- *
- * @param parameters {@link FriendshipUpdateRequest}
- * @return User
- * @see <a href="https://dev.twitter.com/rest/reference/post/friendships/update">https://dev.twitter.com/rest/reference/post/friendships/update</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/update.json")
- public User update(@QueryIfNE("*") FriendshipUpdateRequest parameters);
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Geo.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Geo.java
deleted file mode 100644
index 21f8361..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Geo.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.Tweet;
-import org.apache.streams.twitter.pojo.Place;
-
-import org.apache.juneau.remoteable.Query;
-import org.apache.juneau.remoteable.QueryIfNE;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-
-import java.util.List;
-
-/**
- * Interface for /geo methods.
- *
- * @see <a href="https://dev.twitter.com/rest/reference">https://dev.twitter.com/rest/reference</a>
- */
-@Remoteable(path = "https://api.twitter.com/1.1/geo")
-public interface Geo {
-
- /**
- * Returns all the information about a known place.
- *
- * @param place_id A place in the world. These IDs can be retrieved from geo/reverse_geocode.
- * @return {@link org.apache.streams.twitter.pojo.Place}
- * @see <a href="https://dev.twitter.com/rest/reference/get/geo/id">https://dev.twitter.com/rest/reference/get/geo/id</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/id/{place_id}.json")
- public Place geoid(@Query("place_id") String place_id);
-
- /**
- * Search for places that can be attached to a statuses/update. Given a latitude and a longitude pair, an IP address, or a name, this request will return a list of all the valid places that can be used as the place_id when updating a status.
- *
- * @param parameters A place in the world. These IDs can be retrieved from geo/reverse_geocode.
- * @return {@link org.apache.streams.twitter.api.GeoSearchResponse}
- * @see <a href="https://dev.twitter.com/rest/reference/get/geo/search">https://dev.twitter.com/rest/reference/get/geo/search</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/search.json")
- public GeoSearchResponse geosearch(@QueryIfNE("*") GeoSearchRequest parameters);
-
- /**
- * Given a latitude and a longitude, searches for up to 20 places that can be used as a place_id when updating a status.
- *
- * @param parameters A place in the world. These IDs can be retrieved from geo/reverse_geocode.
- * @return {@link org.apache.streams.twitter.api.GeoSearchResponse}
- * @see <a href="https://dev.twitter.com/rest/reference/get/geo/reverse_geocode">https://dev.twitter.com/rest/reference/get/geo/reverse_geocode</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/reverse_geocode.json")
- public GeoSearchResponse reverseGeocode(@QueryIfNE("*") GeoSearchRequest parameters);
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Media.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Media.java
deleted file mode 100644
index 4460a21..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Media.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.Tweet;
-
-import org.apache.juneau.remoteable.QueryIfNE;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-
-import java.util.List;
-
-/**
- * Interface for /media methods.
- *
- * @see <a href="https://dev.twitter.com/rest/reference">https://dev.twitter.com/rest/reference</a>
- */
-@Remoteable(path = "https://upload.twitter.com/1.1/media")
-public interface Media {
-
- /**
- * The STATUS command is used to periodically poll for updates of media processing operation. After the STATUS command response returns succeeded, you can move on to the next step which is usually create Tweet with media_id.
- *
- * @param parameters {@link org.apache.streams.twitter.api.MediaStatusRequest}
- * @return {@link org.apache.streams.twitter.api.MediaStatusResponse)
- * @see <a href="https://dev.twitter.com/rest/reference/get/media/upload-status">https://dev.twitter.com/rest/reference/get/media/upload-status</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/upload")
- public MediaStatusResponse getMediaStatus(@QueryIfNE MediaStatusRequest parameters );
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Statuses.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Statuses.java
deleted file mode 100644
index 65f3882..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Statuses.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.Tweet;
-
-import org.apache.juneau.remoteable.Body;
-import org.apache.juneau.remoteable.Path;
-import org.apache.juneau.remoteable.Query;
-import org.apache.juneau.remoteable.QueryIfNE;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-import org.apache.juneau.remoteable.RequestBean;
-
-import java.util.List;
-
-/**
- * Interface for /statuses methods.
- *
- * @see <a href="https://dev.twitter.com/rest/reference">https://dev.twitter.com/rest/reference</a>
- */
-@Remoteable(path = "https://api.twitter.com/1.1/statuses")
-public interface Statuses {
-
- /**
- * Returns a collection of the most recent Tweets posted by the user indicated by the screen_name or user_id parameters.
- *
- * @param parameters {@link org.apache.streams.twitter.api.StatusesHomeTimelineRequest}
- * @return List<Tweet>
- * @see <a href="https://dev.twitter.com/rest/reference/get/statuses/home_timeline">https://dev.twitter.com/rest/reference/get/statuses/home_timeline</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/home_timeline.json")
- public List<Tweet> homeTimeline( @QueryIfNE("*") StatusesHomeTimelineRequest parameters );
-
- /**
- * Returns fully-hydrated Tweet objects for up to 100 Tweets per request, as specified by comma-separated values passed to the id parameter.
- *
- * @param parameters {@link org.apache.streams.twitter.api.StatusesLookupRequest}
- * @return List<Tweet>
- * @see <a href="https://dev.twitter.com/rest/reference/get/statuses/lookup">https://dev.twitter.com/rest/reference/get/statuses/lookup</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/lookup.json")
- public List<Tweet> lookup( @QueryIfNE("*") StatusesLookupRequest parameters);
-
- /**
- * Returns the 20 most recent mentions (Tweets containing a users’s @screen_name) for the authenticating user.
- *
- * The timeline returned is the equivalent of the one seen when you view your mentions on twitter.com.
- *
- * This method can only return up to 800 tweets.
- *
- * @param parameters {@link org.apache.streams.twitter.api.StatusesUserTimelineRequest}
- * @return List<Tweet>
- * @see <a href="https://dev.twitter.com/rest/reference/get/statuses/mentions_timeline">https://dev.twitter.com/rest/reference/get/statuses/mentions_timeline</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/mentions_timeline.json")
- public List<Tweet> mentionsTimeline( @QueryIfNE("*") StatusesMentionsTimelineRequest parameters);
-
- /**
- * Returns a single Tweet, specified by the id parameter. The Tweet’s author will also be embedded within the Tweet.
- *
- * @param parameters {@link org.apache.streams.twitter.api.StatusesShowRequest}
- * @return Tweet
- * @see <a href="https://dev.twitter.com/rest/reference/get/statuses/show/id">https://dev.twitter.com/rest/reference/get/statuses/show/id</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/show/{id}")
- public Tweet show( @RequestBean StatusesShowRequest parameters);
-
- /**
- * Returns a collection of the most recent Tweets posted by the user indicated by the screen_name or user_id parameters.
- *
- * @param parameters {@link org.apache.streams.twitter.api.StatusesUserTimelineRequest}
- * @return List<Tweet>
- * @see <a href="https://dev.twitter.com/rest/reference/get/statuses/user_timeline">https://dev.twitter.com/rest/reference/get/statuses/user_timeline</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/user_timeline.json")
- public List<Tweet> userTimeline( @QueryIfNE("*") StatusesUserTimelineRequest parameters);
-
- interface StatusesShowRequestAnnotations {
-
- @Path("id")
- Long getId();
-
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Twitter.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Twitter.java
deleted file mode 100644
index dae6fa8..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Twitter.java
+++ /dev/null
@@ -1,395 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.TwitterConfiguration;
-import org.apache.streams.twitter.converter.TwitterDateTimeFormat;
-import org.apache.streams.twitter.converter.TwitterJodaDateSwap;
-import org.apache.streams.twitter.pojo.DirectMessageEvent;
-import org.apache.streams.twitter.pojo.Tweet;
-import org.apache.streams.twitter.pojo.User;
-import org.apache.streams.twitter.pojo.WelcomeMessage;
-import org.apache.streams.twitter.pojo.WelcomeMessageRule;
-import org.apache.streams.twitter.provider.TwitterProviderUtil;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.http.HttpRequestInterceptor;
-import org.apache.http.HttpResponseInterceptor;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.util.EntityUtils;
-import org.apache.juneau.json.JsonParser;
-import org.apache.juneau.json.JsonSerializer;
-import org.apache.juneau.rest.client.RestCall;
-import org.apache.juneau.rest.client.RestCallException;
-import org.apache.juneau.rest.client.RestClient;
-import org.apache.juneau.rest.client.RestClientBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-//import org.apache.juneau.rest.client.RestClientBuilder;
-
-/**
- * Implementation of all twitter interfaces using juneau.
- */
-public class Twitter implements Account, AccountActivity, DirectMessages, Favorites, Followers, Friends, Statuses, Users, WelcomeMessages, WelcomeMessageRules {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(Twitter.class);
-
- private static Map<TwitterConfiguration, Twitter> INSTANCE_MAP = new ConcurrentHashMap<>();
-
- private TwitterConfiguration configuration;
-
- private ObjectMapper mapper;
-
- private String rootUrl;
-
- private CloseableHttpClient httpclient;
-
- private HttpRequestInterceptor oauthInterceptor;
-
- private static Map<String,Object> properties = new HashMap<String,Object>();
-
- static {
- properties.put("format", TwitterDateTimeFormat.TWITTER_FORMAT);
- }
-
- RestClient restClient;
-
- private Twitter(TwitterConfiguration configuration) throws InstantiationException {
- this.configuration = configuration;
- this.rootUrl = TwitterProviderUtil.baseUrl(configuration);
- this.oauthInterceptor = new TwitterOAuthRequestInterceptor(configuration.getOauth());
- this.httpclient = HttpClientBuilder.create()
- .setDefaultRequestConfig(
- RequestConfig.custom()
- .setConnectionRequestTimeout(5000)
- .setConnectTimeout(5000)
- .setSocketTimeout(5000)
- .setCookieSpec("easy")
- .build()
- )
- .setMaxConnPerRoute(20)
- .setMaxConnTotal(100)
- .addInterceptorFirst(oauthInterceptor)
- .addInterceptorLast((HttpRequestInterceptor) (httpRequest, httpContext) -> LOGGER.debug(httpRequest.getRequestLine().getUri()))
- .addInterceptorLast((HttpResponseInterceptor) (httpResponse, httpContext) -> LOGGER.debug(httpResponse.getStatusLine().toString()))
- .build();
- this.restClient = new RestClientBuilder()
- .debug()
- .httpClient(httpclient, true)
- .parser(
- JsonParser.DEFAULT.builder()
- .ignoreUnknownBeanProperties(true)
- .pojoSwaps(TwitterJodaDateSwap.class)
- .build())
- .serializer(
- JsonSerializer.DEFAULT.builder()
- .pojoSwaps(TwitterJodaDateSwap.class)
- .trimEmptyCollections(true)
- .trimEmptyMaps(true)
- .build())
- .rootUrl(rootUrl)
- .retryable(
- configuration.getRetryMax().intValue(),
- configuration.getRetrySleepMs(),
- new TwitterRetryHandler())
- .build();
- this.mapper = StreamsJacksonMapper.getInstance();
- }
-
- public static Twitter getInstance(TwitterConfiguration configuration) throws InstantiationException {
- if (INSTANCE_MAP.containsKey(configuration) && INSTANCE_MAP.get(configuration) != null) {
- return INSTANCE_MAP.get(configuration);
- } else {
- Twitter twitter = new Twitter(configuration);
- INSTANCE_MAP.put(configuration, twitter);
- return INSTANCE_MAP.get(configuration);
- }
- }
-
- @Override
- public List<Tweet> userTimeline(StatusesUserTimelineRequest parameters) {
- Statuses restStatuses = restClient.getRemoteableProxy(Statuses.class, TwitterProviderUtil.baseUrl(configuration)+"/statuses");
- List<Tweet> result = restStatuses.userTimeline(parameters);
- return result;
- }
-
- @Override
- public List<Tweet> homeTimeline(StatusesHomeTimelineRequest parameters) {
- Statuses restStatuses = restClient.getRemoteableProxy(Statuses.class, TwitterProviderUtil.baseUrl(configuration)+"/statuses");
- List<Tweet> result = restStatuses.homeTimeline(parameters);
- return result;
- }
-
- @Override
- public List<Tweet> lookup(StatusesLookupRequest parameters) {
- Statuses restStatuses = restClient.getRemoteableProxy(Statuses.class, TwitterProviderUtil.baseUrl(configuration)+"/statuses");
- List<Tweet> result = restStatuses.lookup(parameters);
- return result;
- }
-
- @Override
- public List<Tweet> mentionsTimeline(StatusesMentionsTimelineRequest parameters) {
- Statuses restStatuses = restClient.getRemoteableProxy(Statuses.class, TwitterProviderUtil.baseUrl(configuration)+"/statuses");
- List<Tweet> result = restStatuses.mentionsTimeline(parameters);
- return result;
- }
-
- @Override
- public Tweet show(StatusesShowRequest parameters) {
- Statuses restStatuses = restClient.getRemoteableProxy(Statuses.class, TwitterProviderUtil.baseUrl(configuration)+"/statuses");
- Tweet result = restStatuses.show(parameters);
- return result;
- }
-
- @Override
- public FriendsIdsResponse ids(FriendsIdsRequest parameters) {
- Friends restFriends = restClient.getRemoteableProxy(Friends.class, TwitterProviderUtil.baseUrl(configuration)+"/friends");
- FriendsIdsResponse result = restFriends.ids(parameters);
- return result;
- }
-
- @Override
- public FriendsListResponse list(FriendsListRequest parameters) {
- Friends restFriends = restClient.getRemoteableProxy(Friends.class, TwitterProviderUtil.baseUrl(configuration)+"/friends");
- FriendsListResponse result = restFriends.list(parameters);
- return result;
- }
-
- @Override
- public FollowersIdsResponse ids(FollowersIdsRequest parameters) {
- Followers restFollowers = restClient.getRemoteableProxy(Followers.class, TwitterProviderUtil.baseUrl(configuration)+"/followers");
- FollowersIdsResponse result = restFollowers.ids(parameters);
- return result;
- }
-
- @Override
- public FollowersListResponse list(FollowersListRequest parameters) {
- Followers restFollowers = restClient.getRemoteableProxy(Followers.class, TwitterProviderUtil.baseUrl(configuration)+"/followers");
- FollowersListResponse result = restFollowers.list(parameters);
- return result;
- }
-
- @Override
- public List<User> lookup(UsersLookupRequest parameters) {
- Users restUsers = restClient.getRemoteableProxy(Users.class, TwitterProviderUtil.baseUrl(configuration)+"/users");
- List<User> result = restUsers.lookup(parameters);
- return result;
- }
-
- @Override
- public User show(UsersShowRequest parameters) {
- Users restUsers = restClient.getRemoteableProxy(Users.class, TwitterProviderUtil.baseUrl(configuration)+"/users");
- User result = restUsers.show(parameters);
- return result;
- }
-
- @Override
- public List<Tweet> list(FavoritesListRequest parameters) {
- Favorites restFavorites = restClient.getRemoteableProxy(Favorites.class, TwitterProviderUtil.baseUrl(configuration)+"/favorites");
- List<Tweet> result = restFavorites.list(parameters);
- return result;
- }
-
- @Override
- public AccountSettings settings() {
- Account restAccount = restClient.getRemoteableProxy(Account.class, TwitterProviderUtil.baseUrl(configuration)+"/account");
- AccountSettings result = restAccount.settings();
- return result;
- }
-
- @Override
- public User verifyCredentials() {
- Account restAccount = restClient.getRemoteableProxy(Account.class, TwitterProviderUtil.baseUrl(configuration)+"/account");
- User result = restAccount.verifyCredentials();
- return result;
- }
-
- @Override
- public WelcomeMessagesListResponse listWelcomeMessages(WelcomeMessagesListRequest parameters) {
- WelcomeMessages proxy = restClient.getRemoteableProxy(WelcomeMessages.class, TwitterProviderUtil.baseUrl(configuration)+"/direct_messages/welcome_messages");
- return proxy.listWelcomeMessages(parameters);
- }
-
- @Override
- public WelcomeMessage showWelcomeMessage(Long id) {
- WelcomeMessages proxy = restClient.getRemoteableProxy(WelcomeMessages.class, TwitterProviderUtil.baseUrl(configuration)+"/direct_messages/welcome_messages");
- return proxy.showWelcomeMessage(id);
- }
-
- @Override
- public WelcomeMessageNewResponse newWelcomeMessage(WelcomeMessageNewRequest parameters) {
- WelcomeMessages proxy = restClient.getRemoteableProxy(WelcomeMessages.class, TwitterProviderUtil.baseUrl(configuration)+"/direct_messages/welcome_messages");
- return proxy.newWelcomeMessage(parameters);
- }
-
- @Override
- public void destroyWelcomeMessage(Long id) {
- WelcomeMessages proxy = restClient.getRemoteableProxy(WelcomeMessages.class, TwitterProviderUtil.baseUrl(configuration)+"/direct_messages/welcome_messages");
- proxy.destroyWelcomeMessage(id);
- }
-
- @Override
- public WelcomeMessageRulesListResponse listWelcomeMessageRules(WelcomeMessageRulesListRequest parameters) {
- WelcomeMessageRules proxy = restClient.getRemoteableProxy(WelcomeMessageRules.class, TwitterProviderUtil.baseUrl(configuration)+"/direct_messages/welcome_messages/rules");
- return proxy.listWelcomeMessageRules(parameters);
- }
-
- @Override
- public WelcomeMessageRule showWelcomeMessageRule(Long id) {
- WelcomeMessageRules proxy = restClient.getRemoteableProxy(WelcomeMessageRules.class, TwitterProviderUtil.baseUrl(configuration)+"/direct_messages/welcome_messages/rules");
- return proxy.showWelcomeMessageRule(id);
- }
-
- @Override
- public WelcomeMessageRule newWelcomeMessageRule(WelcomeMessageNewRuleRequest body) {
- WelcomeMessageRules proxy = restClient.getRemoteableProxy(WelcomeMessageRules.class, TwitterProviderUtil.baseUrl(configuration)+"/direct_messages/welcome_messages/rules");
- return proxy.newWelcomeMessageRule(body);
- }
-
- @Override
- public void destroyWelcomeMessageRule(Long id) {
- WelcomeMessageRules proxy = restClient.getRemoteableProxy(WelcomeMessageRules.class, TwitterProviderUtil.baseUrl(configuration)+"/direct_messages/welcome_messages/rules");
- proxy.destroyWelcomeMessageRule(id);
- }
-
- @Override
- public List<Webhook> getWebhooks() {
- AccountActivity proxy = restClient.getRemoteableProxy(AccountActivity.class, TwitterProviderUtil.baseUrl(configuration)+"/account_activity");
- return proxy.getWebhooks();
- }
-
- @Override
- public Webhook registerWebhook(String url) {
- AccountActivity proxy = restClient.getRemoteableProxy(AccountActivity.class, TwitterProviderUtil.baseUrl(configuration)+"/account_activity");
- return proxy.registerWebhook(url);
- }
-
- @Override
- public Boolean deleteWebhook(Long webhookId) {
- AccountActivity proxy = restClient.getRemoteableProxy(AccountActivity.class, TwitterProviderUtil.baseUrl(configuration)+"/account_activity");
- return proxy.deleteWebhook(webhookId);
- }
-
- @Override
- public Boolean putWebhook(Long webhookId) {
- AccountActivity proxy = restClient.getRemoteableProxy(AccountActivity.class, TwitterProviderUtil.baseUrl(configuration)+"/account_activity");
- return proxy.putWebhook(webhookId);
- }
-
- @Override
- public Boolean getWebhookSubscription(Long webhookId) {
-// AccountActivity proxy = restClient.getRemoteableProxy(AccountActivity.class, TwitterProviderUtil.baseUrl(configuration)+"/account_activity");
-// return proxy.getWebhookSubscription(webhookId);
- try {
- URIBuilder uriBuilder =
- new URIBuilder()
- .setPath("/account_activity/webhooks/"+webhookId+"/subscriptions.json");
- RestCall restCall = restClient.doGet(uriBuilder.build().toString());
- try {
- int statusCode = restCall
- .getResponse().getStatusLine().getStatusCode();
- return statusCode == 204;
- } catch (RestCallException e) {
- LOGGER.warn("RestCallException", e);
- }
- } catch (IOException e) {
- LOGGER.warn("IOException", e);
- } catch (URISyntaxException e) {
- LOGGER.warn("URISyntaxException", e);
- }
- return false;
- }
-
- @Override
- public Boolean registerWebhookSubscriptions(Long webhookId) {
-// AccountActivity proxy = restClient.getRemoteableProxy(AccountActivity.class, TwitterProviderUtil.baseUrl(configuration)+"/account_activity");
-// return proxy.registerWebhookSubscriptions(webhookId);
- try {
- URIBuilder uriBuilder =
- new URIBuilder()
- .setPath("/account_activity/webhooks/"+webhookId+"/subscriptions.json");
- RestCall restCall = restClient.doPost(uriBuilder.build().toString());
- try {
- int statusCode = restCall.getResponse().getStatusLine().getStatusCode();
- return statusCode == 204;
- } catch (RestCallException e) {
- LOGGER.warn("RestCallException", e);
- }
- } catch (IOException e) {
- LOGGER.warn("IOException", e);
- } catch (URISyntaxException e) {
- LOGGER.warn("URISyntaxException", e);
- }
- return false;
- }
-
- @Override
- public Boolean deleteWebhookSubscriptions(Long webhookId) {
-// AccountActivity proxy = restClient.getRemoteableProxy(AccountActivity.class, TwitterProviderUtil.baseUrl(configuration)+"/account_activity");
-// return proxy.deleteWebhookSubscriptions(webhookId);
- try {
- URIBuilder uriBuilder =
- new URIBuilder()
- .setPath("/account_activity/webhooks/"+webhookId+"/subscriptions.json");
- RestCall restCall = restClient.doDelete(uriBuilder.build().toString());
- try {
- int statusCode = restCall.getResponse().getStatusLine().getStatusCode();
- return statusCode == 204;
- } catch (RestCallException e) {
- LOGGER.warn("RestCallException", e);
- }
- } catch (IOException e) {
- LOGGER.warn("IOException", e);
- } catch (URISyntaxException e) {
- LOGGER.warn("URISyntaxException", e);
- }
- return false;
- }
-
- @Override
- public EventsListResponse listEvents(EventsListRequest parameters) {
- DirectMessages proxy = restClient.getRemoteableProxy(DirectMessages.class, TwitterProviderUtil.baseUrl(configuration)+"/direct_messages");
- return proxy.listEvents(parameters);
- }
-
- @Override
- public EventShowResponse showEvent(Long id) {
- DirectMessages proxy = restClient.getRemoteableProxy(DirectMessages.class, TwitterProviderUtil.baseUrl(configuration)+"/direct_messages");
- return proxy.showEvent(id);
- }
-
- @Override
- public DirectMessageEvent newEvent(MessageCreateRequest event) {
- DirectMessages proxy = restClient.getRemoteableProxy(DirectMessages.class, TwitterProviderUtil.baseUrl(configuration)+"/direct_messages");
- return proxy.newEvent(event);
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/TwitterOAuthRequestInterceptor.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/TwitterOAuthRequestInterceptor.java
deleted file mode 100644
index 615bb33..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/TwitterOAuthRequestInterceptor.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.TwitterOAuthConfiguration;
-
-import org.apache.commons.codec.EncoderException;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpEntityEnclosingRequest;
-import org.apache.http.HttpException;
-import org.apache.http.HttpRequest;
-import org.apache.http.HttpRequestInterceptor;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpRequestWrapper;
-import org.apache.http.protocol.HttpContext;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URLDecoder;
-import java.security.GeneralSecurityException;
-import java.util.Calendar;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-
-/**
- * Handles request signing to api.twitter.com
- */
-public class TwitterOAuthRequestInterceptor implements HttpRequestInterceptor {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterOAuthRequestInterceptor.class);
-
- private static final Base64 base64 = new Base64();
-
- TwitterOAuthConfiguration oAuthConfiguration;
-
- public TwitterSecurity security = new TwitterSecurity();
-
- public TwitterOAuthRequestInterceptor(TwitterOAuthConfiguration oAuthConfiguration) {
- this.oAuthConfiguration = oAuthConfiguration;
- }
-
- public String generateTimestamp() {
- Calendar tempcal = Calendar.getInstance();
- long ts = tempcal.getTimeInMillis();// get current time in milliseconds
- String oauth_timestamp = (new Long(ts/1000)).toString();
- return oauth_timestamp;
- }
-
- public String generateNonce() {
- String uuid_string = UUID.randomUUID().toString();
- uuid_string = uuid_string.replaceAll("-", "");
- String oauth_nonce = base64.encode(uuid_string.getBytes()).toString();
- return oauth_nonce;
- }
-
- @Override
- public void process(HttpRequest httpRequest, HttpContext httpContext) throws HttpException, IOException {
-
- String oauth_nonce = generateNonce();
-
- String oauth_timestamp = generateTimestamp();
-
- Map<String,String> oauthParamMap = new HashMap<>();
- oauthParamMap.put("oauth_consumer_key", oAuthConfiguration.getConsumerKey());
- oauthParamMap.put("oauth_nonce", oauth_nonce);
- oauthParamMap.put("oauth_signature_method", security.oauth_signature_method_param);
- oauthParamMap.put("oauth_timestamp", oauth_timestamp);
- oauthParamMap.put("oauth_token", oAuthConfiguration.getAccessToken());
- oauthParamMap.put("oauth_version", security.oauth_version);
-
- String request_host = ((HttpRequestWrapper)httpRequest).getTarget().toString().replace(":443","");
- String request_path = httpRequest.getRequestLine().getUri();
-
- Map<String,String> allParamsMap = new HashMap<>(oauthParamMap);
-
- if( request_path.contains("?")) {
- request_path = request_path.substring(0, httpRequest.getRequestLine().getUri().indexOf('?'));
- String request_param_line = httpRequest.getRequestLine().getUri().substring(httpRequest.getRequestLine().getUri().indexOf('?')+1);
- String[] request_params = URLDecoder.decode(request_param_line).split("&");
-
- for( String request_param : request_params ) {
- String key = request_param.substring(0, request_param.indexOf('='));
- String value = request_param.substring(request_param.indexOf('=')+1, request_param.length());
- allParamsMap.put(key, value);
- }
- }
-
- if( ((HttpRequestWrapper) httpRequest).getOriginal() instanceof HttpPost) {
- HttpEntity entity = ((HttpEntityEnclosingRequest)((HttpRequestWrapper) httpRequest).getOriginal()).getEntity();
- if( entity != null ) {
- String body = EntityUtils.toString(entity);
- String[] body_params = body.split(",");
- for (String body_param : body_params) {
- body_param = URLDecoder.decode(body_param);
- String key = body_param.substring(0, body_param.indexOf('='));
- String value = body_param.substring(body_param.indexOf('=') + 1, body_param.length());
- allParamsMap.put(key, value);
- }
- }
- }
-
- allParamsMap = security.encodeMap(allParamsMap);
-
-// if( httpRequest instanceof HttpEntityEnclosingRequest ) {
-// HttpEntity entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity();
-// if( entity != null ) {
-// httpRequest.setHeader("Content-Length", Integer.toString(EntityUtils.toString(entity).length()));
-// }
-// }
-
- String signature_parameter_string = security.generateSignatureParameterString(allParamsMap);
-
- String signature_base_string = security.generateSignatureBaseString(((HttpRequestWrapper) httpRequest).getMethod(), request_host+request_path, signature_parameter_string);
-
- String signing_key = security.encode(oAuthConfiguration.getConsumerSecret()) + "&" + security.encode(oAuthConfiguration.getAccessTokenSecret());
-
- String oauth_signature;
- try {
- oauth_signature = security.computeAndEncodeSignature(signature_base_string, signing_key, security.oauth_signature_method);
- } catch (GeneralSecurityException e) {
- LOGGER.warn("GeneralSecurityException", e);
- return;
- }
-
- oauthParamMap.put("oauth_signature",oauth_signature);
-
- String authorization_header_string = security.generateAuthorizationHeaderString(oauthParamMap);
-
- httpRequest.setHeader("Authorization", authorization_header_string);
-
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/TwitterRetryHandler.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/TwitterRetryHandler.java
deleted file mode 100644
index ab6638d..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/TwitterRetryHandler.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.util.api.requests.backoff.AbstractBackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.BackOffException;
-import org.apache.streams.util.api.requests.backoff.impl.LinearTimeBackOffStrategy;
-
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.protocol.HttpContext;
-import org.apache.http.util.EntityUtils;
-import org.apache.juneau.json.JsonParser;
-import org.apache.juneau.parser.ParseException;
-import org.apache.juneau.rest.client.RetryOn;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import javax.ws.rs.core.Response;
-
-/**
- * Handle expected and unexpected exceptions.
- */
-public class TwitterRetryHandler extends RetryOn {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterRetryHandler.class);
-
- protected boolean onResponse(HttpResponse response) {
- if( response == null ) return false;
- LOGGER.debug(response.toString());
- switch(response.getStatusLine().getStatusCode()) {
- case 200: // Response.Status.OK
- case 304: // Response.Status.NOT_MODIFIED
- case 400: // Response.Status.BAD_REQUEST
- return false;
- case 401: // Response.Status.UNAUTHORIZED
- return true;
- case 403: // Response.Status.FORBIDDEN
- case 404: // Response.Status.NOT_FOUND
- case 406: // Response.Status.NOT_ACCEPTABLE
- case 410: // Response.Status.GONE
- return false;
- case 420: // Enhance Your Calm
- case 429: // Too Many Requests
- return true;
- case 500: // Response.Status.INTERNAL_SERVER_ERROR
- case 502: // Bad Gateway
- case 503: // Response.Status.SERVICE_UNAVAILABLE
- case 504: // Gateway Timeout
- return true;
- default:
- return false;
- }
- }
-
- private ResponseErrors parseResponseErrors(HttpResponse response) {
- ResponseErrors responseErrors = null;
- if (response.getEntity() != null) {
- try {
- String responseEntity = EntityUtils.toString(response.getEntity());
- LOGGER.debug(responseEntity);
- responseErrors = JsonParser.DEFAULT.parse(responseEntity, ResponseErrors.class);
- } catch (IOException ex) {
- LOGGER.error("IOException", ex);
- } catch (ParseException ex) {
- LOGGER.error("ParseException", ex);
- }
- }
- return responseErrors;
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/TwitterSecurity.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/TwitterSecurity.java
deleted file mode 100644
index f02ea93..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/TwitterSecurity.java
+++ /dev/null
@@ -1,132 +0,0 @@
-package org.apache.streams.twitter.api;
-
-import org.apache.commons.codec.binary.Base64;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.security.GeneralSecurityException;
-import java.util.Calendar;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.SortedSet;
-import java.util.StringJoiner;
-import java.util.TreeSet;
-import java.util.UUID;
-import javax.crypto.Mac;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
-
-/**
- * Created by sblackmon on 6/15/17.
- */
-public class TwitterSecurity {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterOAuthRequestInterceptor.class);
-
- public static final String oauth_signature_encoding = "UTF-8";
- public static final String oauth_version = "1.0";
-
- public static final String oauth_signature_method_param = "HMAC-SHA1";
- public static final String oauth_signature_method = "HmacSHA1";
- public static final String webhook_signature_method = "HmacSHA256";
-
- private static final Base64 base64 = new Base64();
-
- public String encode(String value)
- {
- String encoded = null;
- try {
- encoded = URLEncoder.encode(value, oauth_signature_encoding);
- } catch (UnsupportedEncodingException ignore) {
- }
- StringBuilder buf = new StringBuilder(encoded.length());
- char focus;
- for (int i = 0; i < encoded.length(); i++) {
- focus = encoded.charAt(i);
- if (focus == '*') {
- buf.append("%2A");
- } else if (focus == '+') {
- buf.append("%20");
- } else if (focus == '%' && (i + 1) < encoded.length()
- && encoded.charAt(i + 1) == '7' && encoded.charAt(i + 2) == 'E') {
- buf.append('~');
- i += 2;
- } else {
- buf.append(focus);
- }
- }
- return buf.toString();
- }
-
- public String generateSignatureParameterString(Map<String, String> allParamsMap) {
-
- SortedSet<String> sortedKeys = new TreeSet<>(allParamsMap.keySet());
-
- StringJoiner stringJoiner = new StringJoiner("&");
- for( String key : sortedKeys ) {
- stringJoiner.add(key+"="+allParamsMap.get(key));
- }
-
- return stringJoiner.toString();
- }
-
- public String generateSignatureBaseString(String method, String request_url, String signature_parameter_string) {
- String signature_base_string = new StringBuilder()
- .append(method)
- .append("&")
- .append(encode(request_url))
- .append("&")
- .append(encode(signature_parameter_string))
- .toString();
- return signature_base_string;
- }
-
- public String generateAuthorizationHeaderString(Map<String,String> oauthParamMap) {
- SortedSet<String> sortedKeys = new TreeSet<>(oauthParamMap.keySet());
-
- StringJoiner stringJoiner = new StringJoiner(", ");
- for( String key : sortedKeys ) {
- stringJoiner.add(encode(key)+"="+"\""+encode(oauthParamMap.get(key))+"\"");
- }
-
- String authorization_header_string = new StringBuilder()
- .append("OAuth ")
- .append(stringJoiner.toString())
- .toString();
- return authorization_header_string;
- }
-
- public Map<String, String> encodeMap(Map<String, String> map) {
- Map<String,String> newMap = new HashMap<>();
- for( String key : map.keySet() ) {
- String value = map.get(key);
- newMap.put(encode(key), encode(value));
- }
- return newMap;
- }
-
- public byte[] computeSignatureBytes(String baseString, String keyString, String signature_method) throws GeneralSecurityException, UnsupportedEncodingException
- {
- SecretKey secretKey = null;
-
- byte[] keyBytes = keyString.getBytes();
- secretKey = new SecretKeySpec(keyBytes, signature_method);
-
- Mac mac = Mac.getInstance(signature_method);
- mac.init(secretKey);
-
- byte[] text = baseString.getBytes();
-
- return mac.doFinal(text);
- }
-
- public String computeAndEncodeSignature(String baseString, String keyString, String signature_method) throws GeneralSecurityException, UnsupportedEncodingException {
-
- return new String(base64.encode(computeSignatureBytes(baseString, keyString, signature_method))).trim();
-
- }
-
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Users.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Users.java
deleted file mode 100644
index a7b1a8f..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/Users.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.Tweet;
-import org.apache.streams.twitter.pojo.User;
-
-import org.apache.juneau.remoteable.Path;
-import org.apache.juneau.remoteable.QueryIfNE;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-import org.apache.juneau.remoteable.RequestBean;
-
-import java.util.List;
-
-/**
- * Interface for /users methods.
- *
- * @see <a href="https://dev.twitter.com/rest/reference">https://dev.twitter.com/rest/reference</a>
- */
-@Remoteable(path = "https://api.twitter.com/1.1/users")
-public interface Users {
-
- /**
- * Returns fully-hydrated user objects for up to 100 users per request, as specified by comma-separated values passed to the user_id and/or screen_name parameters.
- *
- * @param parameters {@link org.apache.streams.twitter.api.UsersLookupRequest}
- * @return List<User>
- * @see <a href="https://dev.twitter.com/rest/reference/get/users/lookup">https://dev.twitter.com/rest/reference/get/users/lookup</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/lookup.json")
- public List<User> lookup( @QueryIfNE UsersLookupRequest parameters);
-
- /**
- * Returns a variety of information about the user specified by the required user_id or screen_name parameter. The author’s most recent Tweet will be returned inline when possible.
- *
- * @param parameters {@link org.apache.streams.twitter.api.UsersShowRequest}
- * @return List<User>
- * @see <a href="https://dev.twitter.com/rest/reference/get/users/show">https://dev.twitter.com/rest/reference/get/users/show</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/show.json")
- public User show( @QueryIfNE UsersShowRequest parameters);
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/WelcomeMessageRules.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/WelcomeMessageRules.java
deleted file mode 100644
index fa14944..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/WelcomeMessageRules.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.WelcomeMessage;
-import org.apache.streams.twitter.pojo.WelcomeMessageRule;
-
-import org.apache.juneau.remoteable.Body;
-import org.apache.juneau.remoteable.Query;
-import org.apache.juneau.remoteable.QueryIfNE;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-
-/**
- * Interface for /direct_messages/welcome_messages/rules methods.
- *
- * @see <a href="https://dev.twitter.com/rest/reference">https://dev.twitter.com/rest/reference</a>
- */
-@Remoteable(path = "https://api.twitter.com/1.1/direct_messages/welcome_messages/rules")
-public interface WelcomeMessageRules {
-
- /**
- * Returns a list of Welcome Message Rules.
- *
- * @return WelcomeMessageRulesListResponse
- * @see <a href="https://dev.twitter.com/rest/reference/get/direct_messages/welcome_messages/rules/list">https://dev.twitter.com/rest/reference/get/direct_messages/welcome_messages/rules/list</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/list.json")
- public WelcomeMessageRulesListResponse listWelcomeMessageRules(@QueryIfNE WelcomeMessageRulesListRequest parameters);
-
- /**
- * Returns a Welcome Message Rule by the given id.
- *
- * @return WelcomeMessage
- * @see <a href="https://dev.twitter.com/rest/reference/get/direct_messages/events/list">https://dev.twitter.com/rest/reference/get/direct_messages/events/list</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/show.json")
- public WelcomeMessageRule showWelcomeMessageRule(@Query("id") Long id);
-
- /**
- * Creates a new Welcome Message Rule that determines which Welcome Message will be shown in a given conversation. Returns the created rule if successful.
- *
- * Requires a JSON POST body and Content-Type header to be set to application/json. Setting Content-Length may also be required if it is not automatically.
- *
- * Additional rule configurations are forthcoming. For the initial beta release, the most recently created Rule will always take precedence, and the assigned Welcome Message will be displayed in the conversation.
- *
- * @return WelcomeMessage
- * @see <a href="https://dev.twitter.com/rest/reference/post/direct_messages/welcome_messages/rules/new">https://dev.twitter.com/rest/reference/post/direct_messages/welcome_messages/rules/new</a>
- *
- */
- @RemoteMethod(httpMethod = "POST", path = "/new.json")
- public WelcomeMessageRule newWelcomeMessageRule(@Body WelcomeMessageNewRuleRequest body);
-
- /**
- * Deletes a Welcome Message Rule by the given id.
- *
- * @see <a href="https://dev.twitter.com/rest/reference/del/direct_messages/welcome_messages/rules/destroy">https://dev.twitter.com/rest/reference/del/direct_messages/welcome_messages/rules/destroy</a>
- *
- */
- @RemoteMethod(httpMethod = "DELETE", path = "/destroy.json")
- public void destroyWelcomeMessageRule(@Query("id") Long id);
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/WelcomeMessages.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/WelcomeMessages.java
deleted file mode 100644
index db4f252..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/api/WelcomeMessages.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package org.apache.streams.twitter.api;
-
-import org.apache.streams.twitter.pojo.WelcomeMessage;
-
-import org.apache.juneau.remoteable.Body;
-import org.apache.juneau.remoteable.Query;
-import org.apache.juneau.remoteable.QueryIfNE;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.apache.juneau.remoteable.Remoteable;
-import org.apache.juneau.remoteable.RequestBean;
-
-/**
- * Interface for /direct_messages/welcome_messages methods.
- *
- * @see <a href="https://dev.twitter.com/rest/reference">https://dev.twitter.com/rest/reference</a>
- */
-@Remoteable(path = "https://api.twitter.com/1.1/direct_messages/welcome_messages")
-public interface WelcomeMessages {
-
- /**
- * Returns a list of Welcome Messages.
- *
- * @return WelcomeMessageListResponse
- * @see <a href="https://dev.twitter.com/rest/reference/get/direct_messages/welcome_messages/list">https://dev.twitter.com/rest/reference/get/direct_messages/welcome_messages/list</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/list.json")
- public WelcomeMessagesListResponse listWelcomeMessages(@QueryIfNE("*") WelcomeMessagesListRequest parameters);
-
- /**
- * Returns a Welcome Message by the given id.
- *
- * @return WelcomeMessage
- * @see <a href="https://dev.twitter.com/rest/reference/get/direct_messages/events/list">https://dev.twitter.com/rest/reference/get/direct_messages/events/list</a>
- *
- */
- @RemoteMethod(httpMethod = "GET", path = "/show.json")
- public WelcomeMessage showWelcomeMessage(@Query("id") Long id);
-
- /**
- * Creates a new Welcome Message that will be stored and sent in the future from the authenticating user in defined circumstances. Returns the message template in the requested format if successful. Supports publishing with the same elements as Direct Messages (e.g. Quick Replies, media attachments).
- *
- * @return WelcomeMessage
- * @see <a href="https://dev.twitter.com/rest/reference/get/direct_messages/events/list">https://dev.twitter.com/rest/reference/get/direct_messages/events/list</a>
- *
- */
- @RemoteMethod(httpMethod = "POST", path = "/new.json")
- public WelcomeMessageNewResponse newWelcomeMessage(@Body WelcomeMessageNewRequest messageNewRequest);
-
- /**
- * Deletes a Welcome Message by the given id.
- *
- * @see <a href="https://dev.twitter.com/rest/reference/del/direct_messages/welcome_messages/destroy">https://dev.twitter.com/rest/reference/del/direct_messages/welcome_messages/destroy</a>
- *
- */
- @RemoteMethod(httpMethod = "DELETE", path = "/destroy.json")
- public void destroyWelcomeMessage(@Query("id") Long id);
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/StreamsTwitterMapper.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/StreamsTwitterMapper.java
deleted file mode 100644
index 671a830..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/StreamsTwitterMapper.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.converter;
-
-import org.apache.streams.data.util.RFC3339Utils;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.converter.util.TwitterActivityUtil;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * This class assist with handling twitter's date-time format during conversion
- *
- * <p/>
- * Deprecated: use StreamsJacksonMapper.getInstance() with TwitterDateTimeFormat on the classpath instead
- */
-@Deprecated
-public class StreamsTwitterMapper extends StreamsJacksonMapper {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterActivityUtil.class);
-
- public static final String TWITTER_FORMAT = "EEE MMM dd HH:mm:ss Z yyyy";
-
- public static final DateTimeFormatter TWITTER_FORMATTER = DateTimeFormat.forPattern(TWITTER_FORMAT);
-
- /**
- * Convert to millis with TWITTER_FORMATTER.
- * @param dateTime dateTime as String
- * @return millis as Long
- */
- public static Long getMillis(String dateTime) {
-
- // this function is for pig which doesn't handle exceptions well
- try {
- return TWITTER_FORMATTER.parseMillis(dateTime);
- } catch ( Exception ex ) {
- return null;
- }
-
- }
-
- private static final StreamsTwitterMapper INSTANCE = new StreamsTwitterMapper();
-
- public static StreamsTwitterMapper getInstance() {
- return INSTANCE;
- }
-
- /**
- * StreamsTwitterMapper constructor.
- */
- public StreamsTwitterMapper() {
- super();
- registerModule(new SimpleModule() {
- {
- addDeserializer(DateTime.class, new StdDeserializer<DateTime>(DateTime.class) {
- @Override
- public DateTime deserialize(JsonParser jpar, DeserializationContext context) throws IOException, JsonProcessingException {
- DateTime result = null;
- try {
- result = TWITTER_FORMATTER.parseDateTime(jpar.getValueAsString());
- } catch ( Exception ignored ) {
- LOGGER.trace("ignored", ignored);
- }
- try {
- result = RFC3339Utils.getInstance().parseToUTC(jpar.getValueAsString());
- } catch ( Exception ignored ) {
- LOGGER.trace("ignored", ignored);
- }
- return result;
- }
- });
- }
- });
-
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterDateTimeFormat.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterDateTimeFormat.java
deleted file mode 100644
index d8da2c1..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterDateTimeFormat.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.converter;
-
-import org.apache.streams.jackson.StreamsDateTimeFormat;
-
-/**
- * TwitterDateTimeFormat.
- */
-public class TwitterDateTimeFormat implements StreamsDateTimeFormat {
-
- public static final String TWITTER_FORMAT = "EEE MMM dd HH:mm:ss Z yyyy";
-
- @Override
- public String getFormat() {
- return TWITTER_FORMAT;
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterDocumentClassifier.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterDocumentClassifier.java
deleted file mode 100644
index 915a8cf..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterDocumentClassifier.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.converter;
-
-import org.apache.streams.data.DocumentClassifier;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Follow;
-import org.apache.streams.twitter.pojo.FriendList;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.Tweet;
-import org.apache.streams.twitter.pojo.User;
-import org.apache.streams.twitter.pojo.UserstreamEvent;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-
-import static org.apache.streams.twitter.converter.TwitterDateTimeFormat.TWITTER_FORMAT;
-
-/**
- * Ensures twitter documents can be converted to Activity.
- */
-public class TwitterDocumentClassifier implements DocumentClassifier {
-
- @Override
- public List<Class> detectClasses(Object document) {
-
- Objects.requireNonNull(document);
-
- ObjectMapper mapper = StreamsJacksonMapper.getInstance(Collections.singletonList(TWITTER_FORMAT));
-
- ObjectNode objectNode;
- try {
- if ( document instanceof String ) {
- objectNode = mapper.readValue((String) document, ObjectNode.class);
- } else if ( document instanceof ObjectNode ) {
- objectNode = (ObjectNode) document;
- } else {
- objectNode = mapper.convertValue(document, ObjectNode.class);
- }
- } catch (IOException ex) {
- return new ArrayList<>();
- }
-
- List<Class> classList = new ArrayList<>();
-
- if ( objectNode.findValue("retweeted_status") != null
- && objectNode.get("retweeted_status") != null) {
- classList.add(Retweet.class);
- } else if ( objectNode.findValue("delete") != null ) {
- classList.add(Delete.class);
- } else if ( objectNode.findValue("friends") != null
- || objectNode.findValue("friends_str") != null ) {
- classList.add(FriendList.class);
- } else if ( objectNode.findValue("target_object") != null ) {
- classList.add(UserstreamEvent.class);
- } else if ( objectNode.findValue("follower") != null
- && objectNode.findValue("followee") != null) {
- classList.add(Follow.class);
- } else if ( objectNode.findValue("location") != null
- && objectNode.findValue("user") == null) {
- classList.add(User.class);
- } else {
- classList.add(Tweet.class);
- }
-
- return classList;
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterFollowActivityConverter.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterFollowActivityConverter.java
deleted file mode 100644
index fccae58..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterFollowActivityConverter.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.converter;
-
-import org.apache.streams.data.ActivityConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.Provider;
-import org.apache.streams.twitter.converter.util.TwitterActivityUtil;
-import org.apache.streams.twitter.pojo.Follow;
-
-import org.apache.commons.lang.NotImplementedException;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public class TwitterFollowActivityConverter implements ActivityConverter<Follow>, Serializable {
-
- public TwitterFollowActivityConverter() {
- }
-
- private static TwitterFollowActivityConverter instance = new TwitterFollowActivityConverter();
-
- public static TwitterFollowActivityConverter getInstance() {
- return instance;
- }
-
- public static Class requiredClass = Follow.class;
-
- @Override
- public Class requiredClass() {
- return requiredClass;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public Follow fromActivity(Activity deserialized) throws ActivityConversionException {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Follow> fromActivityList(List<Activity> list) {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Activity> toActivityList(Follow event) throws ActivityConversionException {
-
- Activity activity = new Activity();
- activity.setVerb("follow");
- activity.setActor(TwitterActivityUtil.buildActor(event.getFollower()));
- activity.setObject(TwitterActivityUtil.buildActor(event.getFollowee()));
- activity.setId(activity.getActor().getId() + "-follow->" + activity.getObject().getId());
- activity.setProvider((Provider) new Provider().withId("twitter"));
- return Stream.of(activity).collect(Collectors.toList());
- }
-
- @Override
- public List<Activity> toActivityList(List<Follow> list) {
- throw new NotImplementedException();
- }
-
-
-}
-
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJodaDateSwap.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJodaDateSwap.java
deleted file mode 100644
index 18f8332..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJodaDateSwap.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-
-package org.apache.streams.twitter.converter;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.juneau.BeanSession;
-import org.apache.juneau.ClassMeta;
-import org.apache.juneau.parser.ParseException;
-import org.apache.juneau.transform.StringSwap;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
-
-/**
- * Transforms {@link org.joda.time.DateTime} to {@link String Strings}.
- */
-
-public class TwitterJodaDateSwap extends StringSwap<DateTime> {
-
- DateTimeFormatter dateFormatter = DateTimeFormat.forPattern(TwitterDateTimeFormat.TWITTER_FORMAT);
-
- @Override /* PojoSwap */
- public String swap(BeanSession session, DateTime o) {
- return dateFormatter.print(o);
- }
-
- @Override /* PojoSwap */
- public DateTime unswap(BeanSession session, String f, ClassMeta<?> hint) throws ParseException {
- if( hint.equals("org.joda.time.DateTime"))
- return dateFormatter.parseDateTime(f);
- else return null;
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonDeleteActivityConverter.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonDeleteActivityConverter.java
deleted file mode 100644
index 0e290ac..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonDeleteActivityConverter.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.converter;
-
-import org.apache.streams.data.ActivityConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Tweet;
-
-import org.apache.commons.lang.NotImplementedException;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.apache.streams.twitter.converter.util.TwitterActivityUtil.updateActivity;
-
-/**
- * TwitterJsonDeleteActivityConverter.
- */
-//TODO: use class explicitly somewhere
-public class TwitterJsonDeleteActivityConverter implements ActivityConverter<Delete>, Serializable {
-
- public static Class requiredClass = Delete.class;
-
- @Override
- public Class requiredClass() {
- return requiredClass;
- }
-
- private static TwitterJsonDeleteActivityConverter instance = new TwitterJsonDeleteActivityConverter();
-
- public static TwitterJsonDeleteActivityConverter getInstance() {
- return instance;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public Delete fromActivity(Activity deserialized) throws ActivityConversionException {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Delete> fromActivityList(List<Activity> list) {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Activity> toActivityList(List<Delete> serializedList) {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Activity> toActivityList(Delete delete) throws ActivityConversionException {
-
- Activity activity = new Activity();
- updateActivity(delete, activity);
- return Stream.of(activity).collect(Collectors.toList());
- }
-
- public ActivityObject buildTarget(Tweet tweet) {
- return null;
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonRetweetActivityConverter.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonRetweetActivityConverter.java
deleted file mode 100644
index feb3e58..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonRetweetActivityConverter.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.converter;
-
-import org.apache.streams.data.ActivityConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.twitter.pojo.Retweet;
-
-import org.apache.commons.lang.NotImplementedException;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.apache.streams.twitter.converter.util.TwitterActivityUtil.updateActivity;
-
-public class TwitterJsonRetweetActivityConverter implements ActivityConverter<Retweet>, Serializable {
-
- public static Class requiredClass = Retweet.class;
-
- @Override
- public Class requiredClass() {
- return requiredClass;
- }
-
- private static TwitterJsonRetweetActivityConverter instance = new TwitterJsonRetweetActivityConverter();
-
- public TwitterJsonRetweetActivityConverter getInstance() {
- return instance;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public Retweet fromActivity(Activity deserialized) throws ActivityConversionException {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Retweet> fromActivityList(List<Activity> list) {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Activity> toActivityList(Retweet retweet) throws ActivityConversionException {
-
- Activity activity = new Activity();
- updateActivity(retweet, activity);
-
- return Stream.of(activity).collect(Collectors.toList());
- }
-
- @Override
- public List<Activity> toActivityList(List<Retweet> serializedList) {
- List<Activity> result = new ArrayList<>();
- for ( Retweet item : serializedList ) {
- try {
- List<Activity> activities = toActivityList(item);
- result.addAll(activities);
- } catch (ActivityConversionException ex) {
- //
- }
- }
- return result;
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonTweetActivityConverter.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonTweetActivityConverter.java
deleted file mode 100644
index d96392f..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonTweetActivityConverter.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.converter;
-
-import org.apache.streams.data.ActivityConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.twitter.pojo.Tweet;
-
-import org.apache.commons.lang.NotImplementedException;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.apache.streams.twitter.converter.util.TwitterActivityUtil.updateActivity;
-
-public class TwitterJsonTweetActivityConverter implements ActivityConverter<Tweet>, Serializable {
-
- public static Class requiredClass = Tweet.class;
-
- @Override
- public Class requiredClass() {
- return requiredClass;
- }
-
- private static TwitterJsonTweetActivityConverter instance = new TwitterJsonTweetActivityConverter();
-
- public static TwitterJsonTweetActivityConverter getInstance() {
- return instance;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public Tweet fromActivity(Activity deserialized) throws ActivityConversionException {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Tweet> fromActivityList(List<Activity> list) {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Activity> toActivityList(Tweet tweet) throws ActivityConversionException {
-
- Activity activity = new Activity();
-
- updateActivity(tweet, activity);
-
- return Stream.of(activity).collect(Collectors.toList());
- }
-
- @Override
- public List<Activity> toActivityList(List<Tweet> serializedList) {
- List<Activity> result = new ArrayList<>();
- for ( Tweet item : serializedList ) {
- try {
- List<Activity> activities = toActivityList(item);
- result.addAll(activities);
- } catch (ActivityConversionException ex) {
- //
- }
- }
- return result;
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonUserActivityConverter.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonUserActivityConverter.java
deleted file mode 100644
index 9ca354f..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonUserActivityConverter.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.converter;
-
-import org.apache.streams.data.ActivityConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.twitter.pojo.User;
-
-import org.apache.commons.lang.NotImplementedException;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import static org.apache.streams.twitter.converter.util.TwitterActivityUtil.updateActivity;
-
-public class TwitterJsonUserActivityConverter implements ActivityConverter<User> {
-
- private static Class requiredClass = User.class;
-
- @Override
- public Class requiredClass() {
- return requiredClass;
- }
-
- private static TwitterJsonUserActivityConverter instance = new TwitterJsonUserActivityConverter();
-
- public static TwitterJsonUserActivityConverter getInstance() {
- return instance;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public User fromActivity(Activity deserialized) throws ActivityConversionException {
- throw new NotImplementedException();
- }
-
- @Override
- public List<User> fromActivityList(List<Activity> list) {
- throw new NotImplementedException();
- }
-
-
- @Override
- public List<Activity> toActivityList(User user) throws ActivityConversionException {
-
- Activity activity = new Activity();
- updateActivity(user, activity);
-
- return Collections.singletonList(activity);
- }
-
- @Override
- public List<Activity> toActivityList(List<User> serializedList) {
- List<Activity> result = new ArrayList<>();
- for ( User item : serializedList ) {
- try {
- List<Activity> activities = toActivityList(item);
- result.addAll(activities);
- } catch (ActivityConversionException ex) {
- //
- }
- }
- return result;
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonUserActivityObjectConverter.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonUserActivityObjectConverter.java
deleted file mode 100644
index 7cb4158..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonUserActivityObjectConverter.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.converter;
-
-import org.apache.streams.data.ActivityObjectConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.twitter.pojo.User;
-
-import org.apache.commons.lang.NotImplementedException;
-
-import static org.apache.streams.twitter.converter.util.TwitterActivityUtil.buildActor;
-
-public class TwitterJsonUserActivityObjectConverter implements ActivityObjectConverter<User> {
-
- public static Class requiredClass = User.class;
-
- @Override
- public Class requiredClass() {
- return requiredClass;
- }
-
- private static TwitterJsonUserActivityObjectConverter instance = new TwitterJsonUserActivityObjectConverter();
-
- public static TwitterJsonUserActivityObjectConverter getInstance() {
- return instance;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public User fromActivityObject(ActivityObject deserialized) throws ActivityConversionException {
- throw new NotImplementedException();
- }
-
- @Override
- public ActivityObject toActivityObject(User serialized) throws ActivityConversionException {
- return buildActor(serialized);
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonUserstreameventActivityConverter.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonUserstreameventActivityConverter.java
deleted file mode 100644
index c96b1a8..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/TwitterJsonUserstreameventActivityConverter.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.converter;
-
-import org.apache.streams.data.ActivityConverter;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.twitter.pojo.UserstreamEvent;
-
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.commons.lang3.StringUtils;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.apache.streams.twitter.converter.util.TwitterActivityUtil.formatId;
-import static org.apache.streams.twitter.converter.util.TwitterActivityUtil.getProvider;
-
-
-/**
- * TwitterJsonUserstreameventActivityConverter.
- */
-// TODO: Use this class explicitely somewhere
-public class TwitterJsonUserstreameventActivityConverter implements ActivityConverter<UserstreamEvent> {
-
- public static Class requiredClass = UserstreamEvent.class;
-
- @Override
- public Class requiredClass() {
- return requiredClass;
- }
-
- private static TwitterJsonUserstreameventActivityConverter instance = new TwitterJsonUserstreameventActivityConverter();
-
- public static TwitterJsonUserstreameventActivityConverter getInstance() {
- return instance;
- }
-
- @Override
- public String serializationFormat() {
- return null;
- }
-
- @Override
- public UserstreamEvent fromActivity(Activity deserialized) throws ActivityConversionException {
- throw new NotImplementedException();
- }
-
- @Override
- public List<UserstreamEvent> fromActivityList(List<Activity> list) {
- throw new NotImplementedException();
- }
-
- @Override
- public List<Activity> toActivityList(UserstreamEvent userstreamEvent) throws ActivityConversionException {
-
- Activity activity = convert(userstreamEvent);
- return Stream.of(activity).collect(Collectors.toList());
-
- }
-
- @Override
- public List<Activity> toActivityList(List<UserstreamEvent> serializedList) {
- return null;
- }
-
- /**
- * convert UserstreamEvent to Activity.
- * @param event UserstreamEvent
- * @return Activity
- * @throws ActivityConversionException ActivityConversionException
- */
- public Activity convert(UserstreamEvent event) throws ActivityConversionException {
-
- Activity activity = new Activity();
- activity.setActor(buildActor(event));
- activity.setVerb(detectVerb(event));
- activity.setObject(buildActivityObject(event));
- activity.setId(formatId(activity.getVerb()));
- if (StringUtils.isEmpty(activity.getId())) {
- throw new ActivityConversionException("Unable to determine activity id");
- }
- activity.setProvider(getProvider());
- return activity;
- }
-
- /**
- * build ActivityObject from UserstreamEvent
- * @param event UserstreamEvent
- * @return $.actor
- */
- public ActivityObject buildActor(UserstreamEvent event) {
- return new ActivityObject();
- }
-
- /**
- * build ActivityObject from UserstreamEvent
- * @param event UserstreamEvent
- * @return $.object
- */
- public ActivityObject buildActivityObject(UserstreamEvent event) {
- return new ActivityObject();
- }
-
- public String detectVerb(UserstreamEvent event) {
- return null;
- }
-
- public ActivityObject buildTarget(UserstreamEvent event) {
- return null;
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/util/TwitterActivityUtil.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/util/TwitterActivityUtil.java
deleted file mode 100644
index 6db16be..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/converter/util/TwitterActivityUtil.java
+++ /dev/null
@@ -1,382 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.converter.util;
-
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.pojo.json.Provider;
-import org.apache.streams.twitter.Url;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Entities;
-import org.apache.streams.twitter.pojo.Hashtag;
-import org.apache.streams.twitter.pojo.Place;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.TargetObject;
-import org.apache.streams.twitter.pojo.Tweet;
-import org.apache.streams.twitter.pojo.User;
-import org.apache.streams.twitter.pojo.UserMentions;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static com.google.common.math.DoubleMath.mean;
-
-/**
- * Provides utilities for working with Activity objects within the context of Twitter.
- */
-public class TwitterActivityUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterActivityUtil.class);
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- /**
- * Updates the given Activity object with the values from the Tweet.
- * @param tweet the object to use as the source
- * @param activity the target of the updates. Will receive all values from the tweet.
- * @throws ActivityConversionException ActivityConversionException
- */
- public static void updateActivity(Tweet tweet, Activity activity) throws ActivityConversionException {
- activity.setActor(buildActor(tweet));
- activity.setId(formatId(activity.getVerb(),
- Optional.ofNullable(Optional.ofNullable(tweet.getIdStr())
- .orElseGet(Optional.of(tweet.getId().toString())::get)).orElse(null)));
-
- if (tweet instanceof Retweet) {
- updateActivityContent(activity, (tweet).getRetweetedStatus(), "share");
- } else {
- updateActivityContent(activity, tweet, "post");
- }
-
- if (StringUtils.isBlank(activity.getId())) {
- throw new ActivityConversionException("Unable to determine activity id");
- }
- try {
- activity.setPublished(tweet.getCreatedAt());
- } catch ( Exception ex ) {
- throw new ActivityConversionException("Unable to determine publishedDate", ex);
- }
- activity.setTarget(buildTarget(tweet));
- activity.setProvider(getProvider());
- activity.setUrl(String.format("http://twitter.com/%s/%s/%s", tweet.getUser().getScreenName(),"/status/",tweet.getIdStr()));
-
- addTwitterExtension(activity, mapper.convertValue(tweet, ObjectNode.class));
- }
-
- /**
- * Updates the given Activity object with the values from the User
- * @param user the object to use as the source
- * @param activity the target of the updates. Will receive all values from the tweet.
- */
- public static void updateActivity(User user, Activity activity) {
- activity.setActor(buildActor(user));
- activity.setId(null);
- activity.setVerb(null);
- }
-
- /**
- * Updates the activity for a delete event.
- * @param delete the delete event
- * @param activity the Activity object to update
- * @throws ActivityConversionException ActivityConversionException
- */
- public static void updateActivity(Delete delete, Activity activity) throws ActivityConversionException {
- activity.setActor(buildActor(delete));
- activity.setVerb("delete");
- activity.setObject(buildActivityObject(delete));
- activity.setId(formatId(activity.getVerb(), delete.getDelete().getStatus().getIdStr()));
- if (StringUtils.isBlank(activity.getId())) {
- throw new ActivityConversionException("Unable to determine activity id");
- }
- activity.setProvider(getProvider());
- addTwitterExtension(activity, StreamsJacksonMapper.getInstance().convertValue(delete, ObjectNode.class));
- }
-
- /**
- * Builds the activity {@link ActivityObject} actor from the tweet.
- * @param tweet the object to use as the source
- * @return a valid Actor populated from the Tweet
- */
- public static ActivityObject buildActor(Tweet tweet) {
- ActivityObject actor = new ActivityObject();
- User user = tweet.getUser();
-
- return buildActor(user);
- }
-
- /**
- * Builds the activity {@link ActivityObject} actor from the User.
- * @param user the object to use as the source
- * @return a valid Actor populated from the Tweet
- */
- public static ActivityObject buildActor(User user) {
- ActivityObject actor = new ActivityObject();
- actor.setId(formatId(
- Optional.ofNullable(Optional.ofNullable(user.getIdStr())
- .orElseGet(Optional.of(user.getId().toString())::get)).orElse(null)
- ));
- actor.setObjectType("page");
- actor.setDisplayName(user.getName());
- actor.setAdditionalProperty("handle", user.getScreenName());
- actor.setSummary(user.getDescription());
-
- if (user.getUrl() != null) {
- actor.setUrl(user.getUrl());
- }
-
- Map<String, Object> extensions = new HashMap<>();
- extensions.put("location", user.getLocation());
- extensions.put("posts", user.getStatusesCount());
- extensions.put("favorites", user.getFavouritesCount());
- extensions.put("followers", user.getFollowersCount());
-
- Image profileImage = new Image();
- profileImage.setUrl(user.getProfileImageUrlHttps());
- actor.setImage(profileImage);
-
- extensions.put("screenName", user.getScreenName());
-
- actor.setAdditionalProperty("extensions", extensions);
- return actor;
- }
-
- /**
- * Builds the actor for a delete event.
- * @param delete the delete event
- * @return a valid Actor
- */
- public static ActivityObject buildActor(Delete delete) {
- ActivityObject actor = new ActivityObject();
- actor.setId(formatId(delete.getDelete().getStatus().getUserIdStr()));
- actor.setObjectType("page");
- return actor;
- }
-
- /**
- * Creates an {@link ActivityObject} for the tweet.
- * @param tweet the object to use as the source
- * @return a valid ActivityObject
- */
- public static ActivityObject buildActivityObject(Tweet tweet) {
- ActivityObject actObj = new ActivityObject();
- String id = Optional.ofNullable(Optional.ofNullable(tweet.getIdStr())
- .orElseGet(Optional.of(tweet.getId().toString())::get)).orElse(null);
- if ( id != null ) {
- actObj.setId(id);
- }
- actObj.setObjectType("post");
- actObj.setContent(tweet.getText());
- return actObj;
- }
-
- /**
- * Builds the ActivityObject for the delete event.
- * @param delete the delete event
- * @return a valid Activity Object
- */
- public static ActivityObject buildActivityObject(Delete delete) {
- ActivityObject actObj = new ActivityObject();
- actObj.setId(formatId(delete.getDelete().getStatus().getIdStr()));
- actObj.setObjectType("tweet");
- return actObj;
- }
-
- /**
- * Updates the content, and associated fields, with those from the given tweet
- * @param activity the target of the updates. Will receive all values from the tweet.
- * @param tweet the object to use as the source
- * @param verb the verb for the given activity's type
- */
- public static void updateActivityContent(Activity activity, Tweet tweet, String verb) {
- activity.setVerb(verb);
- activity.setTitle("");
- if ( tweet != null ) {
- activity.setObject(buildActivityObject(tweet));
- activity.setLinks(getLinks(tweet));
- activity.setContent(tweet.getText());
- addLocationExtension(activity, tweet);
- addTwitterExtensions(activity, tweet);
- }
- }
-
- /**
- * Gets the links from the Twitter event
- * @param tweet the object to use as the source
- * @return a list of links corresponding to the expanded URL (no t.co)
- */
- public static List<String> getLinks(Tweet tweet) {
- List<String> links = new ArrayList<>();
- if ( tweet.getEntities().getUrls() != null ) {
- for (Url url : tweet.getEntities().getUrls()) {
- links.add(url.getExpandedUrl());
- }
- } else {
- LOGGER.debug(" 0 links");
- }
- return links;
- }
-
- /**
- * Builds the {@link TargetObject} from the tweet.
- * @param tweet the object to use as the source
- * @return currently returns null for all activities
- */
- public static ActivityObject buildTarget(Tweet tweet) {
- return null;
- }
-
- /**
- * Adds the location extension and populates with teh twitter data.
- * @param activity the Activity object to update
- * @param tweet the object to use as the source
- */
- public static void addLocationExtension(Activity activity, Tweet tweet) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- Map<String, Object> location = new HashMap<>();
- location.put("id", formatId(
- Optional.ofNullable(Optional.ofNullable(tweet.getIdStr())
- .orElseGet(Optional.of(tweet.getId().toString())::get)).orElse(null)
- ));
- location.put("coordinates", boundingBoxCenter(tweet.getPlace()));
- extensions.put("location", location);
- }
-
- /**
- * Gets the common twitter {@link Provider} object
- * @return a provider object representing Twitter
- */
- public static Provider getProvider() {
- Provider provider = new Provider();
- provider.setId("id:providers:twitter");
- provider.setObjectType("application");
- provider.setDisplayName("Twitter");
- return provider;
- }
-
- /**
- * Adds the given Twitter event to the activity as an extension.
- * @param activity the Activity object to update
- * @param event the Twitter event to add as the extension
- */
- public static void addTwitterExtension(Activity activity, ObjectNode event) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
- extensions.put("twitter", event);
- }
-
- /**
- * Formats the ID to conform with the Apache Streams activity ID convention.
- * @param idparts the parts of the ID to join
- * @return a valid Activity ID in format "id:twitter:part1:part2:...partN"
- */
- public static String formatId(String... idparts) {
- return String.join(":", Stream.concat(Arrays.stream(new String[]{"id:twitter"}), Arrays.stream(idparts)).collect(Collectors.toList()));
- }
-
- /**
- * Takes various parameters from the twitter object that are currently not part of the
- * activity schema and stores them in a generic extensions attribute.
- * @param activity Activity
- * @param tweet Tweet
- */
- public static void addTwitterExtensions(Activity activity, Tweet tweet) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
-
- List<String> hashtags = new ArrayList<>();
- for (Hashtag hashtag : tweet.getEntities().getHashtags()) {
- hashtags.add(hashtag.getText());
- }
- extensions.put("hashtags", hashtags);
-
- Map<String, Object> likes = new HashMap<>();
- likes.put("perspectival", tweet.getFavorited());
- likes.put("count", tweet.getAdditionalProperties().get("favorite_count"));
-
- extensions.put("likes", likes);
-
- Map<String, Object> rebroadcasts = new HashMap<>();
- rebroadcasts.put("perspectival", tweet.getRetweeted());
- rebroadcasts.put("count", tweet.getRetweetCount());
-
- extensions.put("rebroadcasts", rebroadcasts);
-
- List<Map<String, Object>> userMentions = new ArrayList<>();
- Entities entities = tweet.getEntities();
-
- for (UserMentions user : entities.getUserMentions()) {
- //Map the twitter user object into an actor
- Map<String, Object> actor = new HashMap<>();
- actor.put("id", "id:twitter:" + user.getIdStr());
- actor.put("displayName", user.getName());
- actor.put("handle", user.getScreenName());
-
- userMentions.add(actor);
- }
-
- extensions.put("user_mentions", userMentions);
-
- extensions.put("keywords", tweet.getText());
- }
-
- /**
- * Compute central coordinates from bounding box.
- * @param place the bounding box to use as the source
- */
- public static List<Double> boundingBoxCenter(Place place) {
- if ( place == null ) {
- return new ArrayList<>();
- }
- if ( place.getBoundingBox() == null ) {
- return new ArrayList<>();
- }
- if ( place.getBoundingBox().getCoordinates().size() != 1 ) {
- return new ArrayList<>();
- }
- if ( place.getBoundingBox().getCoordinates().get(0).size() != 4 ) {
- return new ArrayList<>();
- }
- List<Double> lats = new ArrayList<>();
- List<Double> lons = new ArrayList<>();
- for ( List<Double> point : place.getBoundingBox().getCoordinates().get(0)) {
- lats.add(point.get(0));
- lons.add(point.get(1));
- }
- List<Double> result = new ArrayList<>();
- result.add(mean(lats));
- result.add(mean(lons));
- return result;
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/FetchAndReplaceTwitterProcessor.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/FetchAndReplaceTwitterProcessor.java
deleted file mode 100644
index ec051fc..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/FetchAndReplaceTwitterProcessor.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.processor;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.twitter.TwitterConfiguration;
-import org.apache.streams.twitter.TwitterStreamConfiguration;
-import org.apache.streams.twitter.api.StatusesShowRequest;
-import org.apache.streams.twitter.api.Twitter;
-import org.apache.streams.twitter.converter.TwitterDocumentClassifier;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.Tweet;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.apache.streams.twitter.converter.util.TwitterActivityUtil.getProvider;
-import static org.apache.streams.twitter.converter.util.TwitterActivityUtil.updateActivity;
-
-/**
- * Given an Activity, fetches the tweet by the activity object id and replaces the existing activity with the converted activity
- * from what is returned by the twitter API.
- */
-public class FetchAndReplaceTwitterProcessor implements StreamsProcessor {
-
- private static final String PROVIDER_ID = getProvider().getId();
- private static final Logger LOGGER = LoggerFactory.getLogger(FetchAndReplaceTwitterProcessor.class);
-
- //Default number of attempts before allowing the document through
- private static final int MAX_ATTEMPTS = 5;
- //Start the backoff at 4 minutes. This results in a wait period of 4, 8, 12, 16 & 20 min with an attempt of 5
- public static final int BACKOFF = 1000 * 60 * 4;
-
- private final TwitterConfiguration config;
- private Twitter client;
- private ObjectMapper mapper;
- private int retryCount;
-
- public FetchAndReplaceTwitterProcessor() {
- this(new ComponentConfigurator<>(TwitterStreamConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig(), "twitter"));
- }
-
- public FetchAndReplaceTwitterProcessor(TwitterStreamConfiguration config) {
- this.config = config;
- }
-
- @Override
- public String getId() {
- return getProvider().getId();
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- if (entry.getDocument() instanceof Activity) {
- Activity doc = (Activity)entry.getDocument();
- String originalId = doc.getId();
- if (PROVIDER_ID.equals(doc.getProvider().getId())) {
- try {
- fetchAndReplace(doc, originalId);
- } catch (ActivityConversionException ex) {
- LOGGER.warn("ActivityConversionException", ex);
- } catch (IOException ex) {
- LOGGER.warn("IOException", ex);
- }
- }
- } else {
- throw new IllegalStateException("Requires an activity document");
- }
- return Stream.of(entry).collect(Collectors.toList());
- }
-
-
- @Override
- public void prepare(Object configurationObject) {
- try {
- client = getTwitterClient();
- } catch (InstantiationException e) {
- LOGGER.error("InstantiationException", e);
- }
- Objects.requireNonNull(client);
- this.mapper = StreamsJacksonMapper.getInstance();
- Objects.requireNonNull(mapper);
- }
-
- @Override
- public void cleanUp() {
-
- }
-
- protected void fetchAndReplace(Activity doc, String originalId) throws java.io.IOException, ActivityConversionException {
- Tweet tweet = fetch(doc);
- replace(doc, tweet);
- doc.setId(originalId);
- }
-
- protected void replace(Activity doc, Tweet tweet) throws java.io.IOException, ActivityConversionException {
- String json = mapper.writeValueAsString(tweet);
- Class documentSubType = new TwitterDocumentClassifier().detectClasses(json).get(0);
- Object object = mapper.readValue(json, documentSubType);
-
- if (documentSubType.equals(Retweet.class) || documentSubType.equals(Tweet.class)) {
- updateActivity((Tweet)object, doc);
- } else if (documentSubType.equals(Delete.class)) {
- updateActivity((Delete)object, doc);
- } else {
- LOGGER.info("Could not determine the correct update method for {}", documentSubType);
- }
- }
-
- protected Tweet fetch(Activity doc) {
- String id = doc.getObject().getId();
- LOGGER.debug("Fetching status from Twitter for {}", id);
- Long tweetId = Long.valueOf(id.replace("id:twitter:tweets:", ""));
- Tweet tweet = client.show(new StatusesShowRequest().withId(tweetId));
- return tweet;
- }
-
-
- protected Twitter getTwitterClient() throws InstantiationException {
-
- return Twitter.getInstance(config);
-
- }
-
- //Hardcore sleep to allow for catch up
-// protected void sleepAndTryAgain(Activity doc, String originalId) {
-// try {
-// //Attempt to fetchAndReplace with a backoff up to the limit then just reset the count and let the process continue
-// if (retryCount < MAX_ATTEMPTS) {
-// retryCount++;
-// LOGGER.debug("Sleeping for {} min due to excessive calls to Twitter API", (retryCount * 4));
-// Thread.sleep(BACKOFF * retryCount);
-// fetchAndReplace(doc, originalId);
-// } else {
-// retryCount = 0;
-// }
-// } catch (InterruptedException ex) {
-// LOGGER.warn("Thread sleep interrupted while waiting for twitter backoff");
-// }
-// }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java
deleted file mode 100644
index d51e4e7..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.processor;
-
-import org.apache.streams.converter.ActivityConverterProcessor;
-
-/**
- * This class performs conversion of a twitter event to a specified outClass.
- *
- * <p/>
- * Deprecated: use TypeConverterProcessor and ActivityConverterProcessor instead
- */
-@Deprecated
-public class TwitterTypeConverter extends ActivityConverterProcessor {
-
- public static final String STREAMS_ID = "TwitterTypeConverter";
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
deleted file mode 100644
index c8d439d..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterUrlApiProcessor.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.processor;
-
-import org.apache.streams.components.http.HttpProcessorConfiguration;
-import org.apache.streams.components.http.processor.SimpleHTTPGetProcessor;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.pojo.json.Activity;
-
-import com.google.common.base.Preconditions;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Class gets a global share count from Twitter API for links on Activity datums.
- */
-public class TwitterUrlApiProcessor extends SimpleHTTPGetProcessor implements StreamsProcessor {
-
- private static final String STREAMS_ID = "TwitterUrlApiProcessor";
-
- /**
- * TwitterUrlApiProcessor constructor.
- */
- public TwitterUrlApiProcessor() {
- super();
- this.configuration.setHostname("urls.api.twitter.com");
- this.configuration.setResourcePath("/1/urls/count.json");
- this.configuration.setEntity(HttpProcessorConfiguration.Entity.ACTIVITY);
- this.configuration.setExtension("twitter_url_count");
- }
-
- /**
- * TwitterUrlApiProcessor constructor.
- */
- public TwitterUrlApiProcessor(HttpProcessorConfiguration processorConfiguration) {
- super(processorConfiguration);
- this.configuration.setHostname("urls.api.twitter.com");
- this.configuration.setResourcePath("/1/urls/count.json");
- this.configuration.setEntity(HttpProcessorConfiguration.Entity.ACTIVITY);
- this.configuration.setExtension("twitter_url_count");
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- Preconditions.checkArgument(entry.getDocument() instanceof Activity);
- Activity activity = mapper.convertValue(entry.getDocument(), Activity.class);
- if ( activity.getLinks() != null && activity.getLinks().size() > 0) {
- return super.process(entry);
- } else {
- return Stream.of(entry).collect(Collectors.toList());
- }
- }
-
- @Override
- protected Map<String, String> prepareParams(StreamsDatum entry) {
-
- Map<String, String> params = new HashMap<>();
-
- params.put("url", mapper.convertValue(entry.getDocument(), Activity.class).getLinks().get(0));
-
- return params;
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowersIdsProviderTask.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowersIdsProviderTask.java
deleted file mode 100644
index 7acbeef..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowersIdsProviderTask.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.api.FollowersIdsRequest;
-import org.apache.streams.twitter.api.FollowersIdsResponse;
-import org.apache.streams.twitter.api.Twitter;
-import org.apache.streams.twitter.pojo.Follow;
-import org.apache.streams.twitter.pojo.User;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Retrieve friend or follower connections for a single user id.
- */
-public class TwitterFollowersIdsProviderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterFollowersIdsProviderTask.class);
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- protected Twitter client;
- protected TwitterFollowingProvider provider;
- protected FollowersIdsRequest request;
-
- /**
- * TwitterFollowingProviderTask constructor.
- * @param provider TwitterFollowingProvider
- * @param twitter Twitter
- * @param request FollowersIdsRequest
- */
- public TwitterFollowersIdsProviderTask(TwitterFollowingProvider provider, Twitter twitter, FollowersIdsRequest request) {
- this.provider = provider;
- this.client = twitter;
- this.request = request;
- }
-
- @Override
- public void run() {
-
- Preconditions.checkArgument(request.getId() != null || request.getScreenName() != null);
-
- LOGGER.info("Thread Starting: {}", request.toString());
-
- getFollowersIds(request);
-
- LOGGER.info("Thread Finished: {}", request.toString());
-
- }
-
- int last_count = 0;
- int page_count = 0;
- int item_count = 0;
- long cursor = 0;
-
- private void getFollowersIds(FollowersIdsRequest request) {
-
- do {
-
- FollowersIdsResponse response = client.ids(request);
-
- last_count = response.getIds().size();
-
- if (response.getIds().size() > 0) {
-
- for (Long id : response.getIds()) {
-
- Follow follow = new Follow()
- .withFollowee(
- new User()
- .withId(request.getId())
- .withScreenName(request.getScreenName()))
- .withFollower(
- new User()
- .withId(id));
-
- if (item_count < provider.getConfig().getMaxItems()) {
- ComponentUtils.offerUntilSuccess(new StreamsDatum(follow), provider.providerQueue);
- item_count++;
- }
-
- }
-
- }
- page_count++;
- cursor = response.getNextCursor();
- request.setCursor(cursor);
- }
- while (shouldContinuePulling(cursor, last_count, page_count, item_count));
-
- LOGGER.info("item_count: {} last_count: {} page_count: {} ", item_count, last_count, page_count);
-
- }
-
- public boolean shouldContinuePulling(long cursor, int count, int page_count, int item_count) {
- return (
- cursor > 0
- && count > 0
- && item_count < provider.getConfig().getMaxItems()
- && page_count <= provider.getConfig().getMaxPages());
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowersListProviderTask.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowersListProviderTask.java
deleted file mode 100644
index 5195a13..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowersListProviderTask.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.api.FollowersListRequest;
-import org.apache.streams.twitter.api.FollowersListResponse;
-import org.apache.streams.twitter.api.Twitter;
-import org.apache.streams.twitter.pojo.Follow;
-import org.apache.streams.twitter.pojo.User;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Retrieve friend or follower connections for a single user id.
- */
-public class TwitterFollowersListProviderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterFollowersListProviderTask.class);
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- protected Twitter client;
- protected TwitterFollowingProvider provider;
- protected FollowersListRequest request;
-
- /**
- * TwitterFollowersListProviderTask constructor.
- * @param provider TwitterFollowingProvider
- * @param twitter Twitter
- * @param request FollowersListRequest
- */
- public TwitterFollowersListProviderTask(TwitterFollowingProvider provider, Twitter twitter, FollowersListRequest request) {
- this.provider = provider;
- this.client = twitter;
- this.request = request;
- }
-
- @Override
- public void run() {
-
- LOGGER.info("Thread Starting: {}", request.toString());
-
- getFollowersList(request);
-
- LOGGER.info("Thread Finished: {}", request.toString());
-
- }
-
- int last_count = 0;
- int page_count = 0;
- int item_count = 0;
- long cursor = 0;
-
- private void getFollowersList(FollowersListRequest request) {
-
- do {
-
- FollowersListResponse response = client.list(request);
-
- last_count = response.getUsers().size();
-
- if (response.getUsers().size() > 0) {
-
- for (User follower : response.getUsers()) {
-
- Follow follow = new Follow()
- .withFollowee(
- new User()
- .withId(request.getId())
- .withScreenName(request.getScreenName()))
- .withFollower(follower);
-
- if (item_count < provider.getConfig().getMaxItems()) {
- ComponentUtils.offerUntilSuccess(new StreamsDatum(follow), provider.providerQueue);
- item_count++;
- }
-
- }
-
- }
- page_count++;
- cursor = response.getNextCursor();
- request.setCursor(cursor);
-
- }
- while (shouldContinuePulling(cursor, last_count, page_count, item_count));
-
- LOGGER.info("item_count: {} last_count: {} page_count: {} ", item_count, last_count, page_count);
-
- }
-
- public boolean shouldContinuePulling(long cursor, int count, int page_count, int item_count) {
- return (
- cursor > 0
- && count > 0
- && item_count < provider.getConfig().getMaxItems()
- && page_count <= provider.getConfig().getMaxPages());
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProvider.java
deleted file mode 100644
index 0a2c53b..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProvider.java
+++ /dev/null
@@ -1,367 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.StreamsJacksonMapperConfiguration;
-import org.apache.streams.twitter.TwitterFollowingConfiguration;
-import org.apache.streams.twitter.TwitterUserInformationConfiguration;
-import org.apache.streams.twitter.api.FollowersIdsRequest;
-import org.apache.streams.twitter.api.FollowersListRequest;
-import org.apache.streams.twitter.api.FollowingIdsRequest;
-import org.apache.streams.twitter.api.FriendsIdsRequest;
-import org.apache.streams.twitter.api.FriendsListRequest;
-import org.apache.streams.twitter.api.Twitter;
-import org.apache.streams.twitter.converter.TwitterDateTimeFormat;
-import org.apache.streams.twitter.pojo.User;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * Retrieve all follow adjacencies from a list of user ids or names.
- */
-public class TwitterFollowingProvider implements StreamsProvider, Serializable {
-
- public static final String STREAMS_ID = "TwitterFollowingProvider";
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterFollowingProvider.class);
-
- private final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- private TwitterFollowingConfiguration config;
-
- protected List<String> names = new ArrayList<>();
- protected List<Long> ids = new ArrayList<>();
-
- protected Twitter client;
-
- protected ListeningExecutorService executor;
-
- private List<ListenableFuture<Object>> futures = new ArrayList<>();
-
- protected final AtomicBoolean running = new AtomicBoolean();
-
- protected volatile Queue<StreamsDatum> providerQueue;
-
- /**
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * twitter.oauth.consumerKey
- * twitter.oauth.consumerSecret
- * twitter.oauth.accessToken
- * twitter.oauth.accessTokenSecret
- * twitter.info
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterFollowingProvider -Dexec.args="application.conf tweets.json"
- *
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- File file = new File(configfile);
- assert (file.exists());
- Config testConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false)).withFallback(StreamsConfigurator.getConfig()).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(testConfig);
- TwitterFollowingConfiguration config = new ComponentConfigurator<>(TwitterFollowingConfiguration.class).detectConfiguration(testConfig, "twitter");
- TwitterFollowingProvider provider = new TwitterFollowingProvider(config);
-
- StreamsJacksonMapperConfiguration mapperConfiguration = new StreamsJacksonMapperConfiguration()
- .withDateFormats(Collections.singletonList(TwitterDateTimeFormat.TWITTER_FORMAT));
- ObjectMapper mapper = StreamsJacksonMapper.getInstance(mapperConfiguration);
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Thread.sleep(streamsConfiguration.getBatchFrequencyMs());
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- try {
- json = mapper.writeValueAsString(datum.getDocument());
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-
- public TwitterFollowingConfiguration getConfig() {
- return config;
- }
-
- public static final int MAX_NUMBER_WAITING = 10000;
-
- public TwitterFollowingProvider() {
- this.config = new ComponentConfigurator<>(TwitterFollowingConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig().getConfig("twitter"));
- }
-
- public TwitterFollowingProvider(TwitterFollowingConfiguration config) {
- this.config = config;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- public void prepare(Object configurationObject) {
-
- Objects.requireNonNull(config);
- Objects.requireNonNull(config.getOauth());
- Objects.requireNonNull(config.getOauth().getConsumerKey());
- Objects.requireNonNull(config.getOauth().getConsumerSecret());
- Objects.requireNonNull(config.getOauth().getAccessToken());
- Objects.requireNonNull(config.getOauth().getAccessTokenSecret());
- Objects.requireNonNull(config.getInfo());
- Objects.requireNonNull(config.getThreadsPerProvider());
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
-
- try {
- client = getTwitterClient();
- } catch (InstantiationException e) {
- LOGGER.error("InstantiationException", e);
- }
-
- Objects.requireNonNull(client);
-
- try {
- lock.writeLock().lock();
- providerQueue = constructQueue();
- } finally {
- lock.writeLock().unlock();
- }
-
- Objects.requireNonNull(providerQueue);
-
- // abstract this out
- for (String s : config.getInfo()) {
- if (s != null) {
- String potentialScreenName = s.replaceAll("@", "").trim().toLowerCase();
-
- // See if it is a long, if it is, add it to the user iD list, if it is not, add it to the
- // screen name list
- try {
- ids.add(Long.parseLong(potentialScreenName));
- } catch (Exception ex) {
- names.add(potentialScreenName);
- }
- }
- }
-
- Objects.requireNonNull(getConfig().getEndpoint());
-
- executor = MoreExecutors.listeningDecorator(
- TwitterUserInformationProvider.newFixedThreadPoolWithQueueSize(
- config.getThreadsPerProvider().intValue(),
- streamsConfiguration.getQueueSize().intValue()
- )
- );
-
- Preconditions.checkArgument(getConfig().getEndpoint().equals("friends") || getConfig().getEndpoint().equals("followers"));
-
- for (Long id : ids) {
- submitTask(createTask(id, null));
- LOGGER.info("Thread Submitted: {}", id);
- }
-
- for (String name : names) {
- submitTask(createTask(null, name));
- LOGGER.info("Thread Submitted: {}", name);
- }
-
- }
-
- public void startStream() {
-
- Objects.requireNonNull(executor);
-
- LOGGER.info("startStream");
-
- running.set(true);
-
- LOGGER.info("isRunning");
-
- executor.shutdown();
-
- }
-
- protected Runnable createTask(Long id, String name) {
- if( config.getEndpoint().equals("friends") && config.getIdsOnly() == true ) {
- FriendsIdsRequest request = (FriendsIdsRequest)new FriendsIdsRequest().withId(id).withScreenName(name);
- return new TwitterFriendsIdsProviderTask(
- this,
- client,
- request);
- } else if( config.getEndpoint().equals("friends") && config.getIdsOnly() == false ) {
- FriendsListRequest request = (FriendsListRequest)new FriendsListRequest().withId(id).withScreenName(name);
- return new TwitterFriendsListProviderTask(
- this,
- client,
- request);
- } else if( config.getEndpoint().equals("followers") && config.getIdsOnly() == true ) {
- FollowersIdsRequest request = (FollowersIdsRequest)new FollowersIdsRequest().withId(id).withScreenName(name);
- return new TwitterFollowersIdsProviderTask(
- this,
- client,
- request);
- } else if( config.getEndpoint().equals("followers") && config.getIdsOnly() == false ) {
- FollowersListRequest request = (FollowersListRequest)new FollowersListRequest().withId(id).withScreenName(name);
- return new TwitterFollowersListProviderTask(
- this,
- client,
- request);
- } else return null;
- }
-
- protected void submitTask(Runnable providerTask) {
- ListenableFuture future = executor.submit(providerTask);
- futures.add(future);
- }
-
- protected Twitter getTwitterClient() throws InstantiationException {
- return Twitter.getInstance(config);
- }
-
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet result;
-
- try {
- lock.writeLock().lock();
- result = new StreamsResultSet(providerQueue);
- result.setCounter(new DatumStatusCounter());
- providerQueue = constructQueue();
- LOGGER.debug("readCurrent: {} Documents", result.size());
- } finally {
- lock.writeLock().unlock();
- }
-
- return result;
-
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- throw new NotImplementedException();
- }
-
- public boolean shouldContinuePulling(List<User> users) {
- return (users != null) && (users.size() == config.getPageSize());
- }
-
- public boolean isRunning() {
- if ( providerQueue.isEmpty() && executor.isTerminated() && Futures.allAsList(futures).isDone() ) {
- LOGGER.info("All Threads Completed");
- running.set(false);
- LOGGER.info("Exiting");
- }
- return running.get();
- }
-
- // abstract this out
- protected Queue<StreamsDatum> constructQueue() {
- return new LinkedBlockingQueue<>();
- }
-
- // abstract this out
- void shutdownAndAwaitTermination(ExecutorService pool) {
- pool.shutdown(); // Disable new tasks from being submitted
- try {
- // Wait a while for existing tasks to terminate
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- pool.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- System.err.println("Pool did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- pool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
- public void cleanUp() {
- shutdownAndAwaitTermination(executor);
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFriendsIdsProviderTask.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFriendsIdsProviderTask.java
deleted file mode 100644
index 5a53123..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFriendsIdsProviderTask.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.api.FriendsIdsRequest;
-import org.apache.streams.twitter.api.FriendsIdsResponse;
-import org.apache.streams.twitter.api.Twitter;
-import org.apache.streams.twitter.pojo.Follow;
-import org.apache.streams.twitter.pojo.User;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Retrieve friend or follower connections for a single user id.
- */
-public class TwitterFriendsIdsProviderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterFriendsIdsProviderTask.class);
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- protected Twitter client;
- protected TwitterFollowingProvider provider;
- protected FriendsIdsRequest request;
-
- /**
- * TwitterFollowingProviderTask constructor.
- * @param provider TwitterFollowingProvider
- * @param twitter Twitter
- * @param request FriendsIdsRequest
- */
- public TwitterFriendsIdsProviderTask(TwitterFollowingProvider provider, Twitter twitter, FriendsIdsRequest request) {
- this.provider = provider;
- this.client = twitter;
- this.request = request;
- }
-
- @Override
- public void run() {
-
- Preconditions.checkArgument(request.getId() != null || request.getScreenName() != null);
-
- LOGGER.info("Thread Starting: {}", request.toString());
-
- getFriendsIds(request);
-
- LOGGER.info("Thread Finished: {}", request.toString());
-
- }
-
- int last_count = 0;
- int page_count = 0;
- int item_count = 0;
- long cursor = 0;
-
- private void getFriendsIds(FriendsIdsRequest request) {
-
- do {
-
- FriendsIdsResponse response = client.ids(request);
-
- last_count = response.getIds().size();
-
- if (response.getIds().size() > 0) {
-
- for (Long id : response.getIds()) {
-
- Follow follow = new Follow()
- .withFollowee(
- new User()
- .withId(id))
- .withFollower(
- new User()
- .withId(request.getId())
- .withScreenName(request.getScreenName()));
-
- if (item_count < provider.getConfig().getMaxItems()) {
- ComponentUtils.offerUntilSuccess(new StreamsDatum(follow), provider.providerQueue);
- item_count++;
- }
-
- }
-
- }
- page_count++;
- cursor = response.getNextCursor();
- request.setCursor(cursor);
-
- }
- while (shouldContinuePulling(cursor, last_count, page_count, item_count));
-
- LOGGER.info("item_count: {} last_count: {} page_count: {} ", item_count, last_count, page_count);
-
- }
-
- public boolean shouldContinuePulling(long cursor, int count, int page_count, int item_count) {
- return (
- cursor > 0
- && count > 0
- && item_count < provider.getConfig().getMaxItems()
- && page_count <= provider.getConfig().getMaxPages());
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFriendsListProviderTask.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFriendsListProviderTask.java
deleted file mode 100644
index e116f2f..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFriendsListProviderTask.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.api.FriendsListRequest;
-import org.apache.streams.twitter.api.FriendsListResponse;
-import org.apache.streams.twitter.api.Twitter;
-import org.apache.streams.twitter.pojo.Follow;
-import org.apache.streams.twitter.pojo.User;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Retrieve friend or follower connections for a single user id.
- */
-public class TwitterFriendsListProviderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterFriendsListProviderTask.class);
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- protected Twitter client;
- protected TwitterFollowingProvider provider;
- protected FriendsListRequest request;
-
- /**
- * TwitterFollowingProviderTask constructor.
- * @param provider TwitterFollowingProvider
- * @param twitter Twitter
- * @param request FriendsListRequest
- */
- public TwitterFriendsListProviderTask(TwitterFollowingProvider provider, Twitter twitter, FriendsListRequest request) {
- this.provider = provider;
- this.client = twitter;
- this.request = request;
- }
-
- int last_count = 0;
- int page_count = 0;
- int item_count = 0;
- long cursor = 0;
-
- @Override
- public void run() {
-
- Preconditions.checkArgument(request.getId() != null || request.getScreenName() != null);
-
- getFriendsList(request);
-
- LOGGER.info(request.getId() != null ? request.getId().toString() : request.getScreenName() + " Thread Finished");
-
- }
-
- private void getFriendsList(FriendsListRequest request) {
-
- do {
-
- FriendsListResponse response = client.list(request);
-
- last_count = response.getUsers().size();
-
- if (response.getUsers().size() > 0) {
-
- for (User friend : response.getUsers()) {
-
- Follow follow = new Follow()
- .withFollower(
- new User()
- .withId(request.getId())
- .withScreenName(request.getScreenName()))
- .withFollowee(
- friend
- );
-
- if (item_count < provider.getConfig().getMaxItems()) {
- ComponentUtils.offerUntilSuccess(new StreamsDatum(follow), provider.providerQueue);
- item_count++;
- }
-
- }
-
- }
- page_count++;
- cursor = response.getNextCursor();
- request.setCursor(cursor);
-
- }
- while (shouldContinuePulling(cursor, last_count, page_count, item_count));
-
- LOGGER.info("item_count: {} last_count: {} page_count: {} ", item_count, last_count, page_count);
-
- }
-
- public boolean shouldContinuePulling(long cursor, int count, int page_count, int item_count) {
- return (
- cursor > 0
- && count > 0
- && item_count < provider.getConfig().getMaxItems()
- && page_count <= provider.getConfig().getMaxPages());
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterProviderUtil.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterProviderUtil.java
deleted file mode 100644
index 56d735d..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterProviderUtil.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.provider;
-
-import org.apache.streams.twitter.TwitterConfiguration;
-
-/**
- * TwitterProviderUtil contains utilities for Twitter Providers.
- */
-public class TwitterProviderUtil {
-
- /**
- * baseUrl from TwitterConfiguration.
- * @param config TwitterConfiguration
- * @return baseUrl
- */
- public static String baseUrl(TwitterConfiguration config) {
-
- String baseUrl = new StringBuilder()
- .append(config.getProtocol())
- .append("://")
- .append(config.getHost())
- .append(":")
- .append(config.getPort())
- .append("/")
- .append(config.getVersion())
- .toString();
-
- return baseUrl;
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamHelper.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamHelper.java
deleted file mode 100644
index 2be77ac..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamHelper.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.converter.TwitterDocumentClassifier;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.twitter.hbc.core.processor.StringDelimitedProcessor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-/**
- * TwitterStreamHelper helps with hosebird twitter stream.
- */
-public class TwitterStreamHelper extends StringDelimitedProcessor {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterStreamHelper.class);
- private static final int DEFAULT_POOL_SIZE = 5;
-
- private static final TwitterDocumentClassifier TWITTER_DOCUMENT_CLASSIFIER = new TwitterDocumentClassifier();
-
- private final TwitterStreamProvider provider;
- private final ExecutorService service;
-
- public TwitterStreamHelper(TwitterStreamProvider provider) {
- this(provider, DEFAULT_POOL_SIZE);
- }
-
- /**
- * TwitterStreamHelper constructor.
- * @param provider TwitterStreamProvider
- * @param poolSize poolSize
- */
- public TwitterStreamHelper(TwitterStreamProvider provider, int poolSize) {
- //We are only going to use the Hosebird processor to manage the extraction of the tweets from the Stream
- super(null);
- service = Executors.newFixedThreadPool(poolSize);
- this.provider = provider;
- }
-
- @Override
- public boolean process() throws IOException, InterruptedException {
- String msg;
- do {
- msg = this.processNextMessage();
- if (msg == null) {
- Thread.sleep(10);
- }
- }
- while (msg == null);
-
- //Deserializing to an ObjectNode can take time. Parallelize the task to improve throughput
- return provider.addDatum(service.submit(new StreamDeserializer(msg)));
- }
-
- public void cleanUp() {
- ComponentUtils.shutdownExecutor(service, 1, 30);
- }
-
- protected static class StreamDeserializer implements Callable<List<StreamsDatum>> {
-
- protected static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- protected String item;
-
- public StreamDeserializer(String item) {
- this.item = item;
- }
-
- @Override
- public List<StreamsDatum> call() throws Exception {
- if (item != null) {
- Class itemClass = TWITTER_DOCUMENT_CLASSIFIER.detectClasses(item).get(0);
- Object document = mapper.readValue(item, itemClass);
- StreamsDatum rawDatum = new StreamsDatum(document);
- List<StreamsDatum> streamsDatumList = new ArrayList<>();
- streamsDatumList.add(rawDatum);
- return streamsDatumList;
- }
- return new ArrayList<>();
- }
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
deleted file mode 100644
index 263e3b6..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatus;
-import org.apache.streams.core.DatumStatusCountable;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.TwitterStreamConfiguration;
-import org.apache.streams.twitter.converter.TwitterDateTimeFormat;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.twitter.hbc.ClientBuilder;
-import com.twitter.hbc.core.Constants;
-import com.twitter.hbc.core.Hosts;
-import com.twitter.hbc.core.HttpHosts;
-import com.twitter.hbc.core.endpoint.StatusesFilterEndpoint;
-import com.twitter.hbc.core.endpoint.StatusesFirehoseEndpoint;
-import com.twitter.hbc.core.endpoint.StatusesSampleEndpoint;
-import com.twitter.hbc.core.endpoint.StreamingEndpoint;
-import com.twitter.hbc.core.endpoint.UserstreamEndpoint;
-import com.twitter.hbc.httpclient.BasicClient;
-import com.twitter.hbc.httpclient.auth.Authentication;
-import com.twitter.hbc.httpclient.auth.BasicAuth;
-import com.twitter.hbc.httpclient.auth.OAuth1;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * TwitterStreamProvider wraps a hosebird client and passes received documents
- * to subscribing components.
- */
-public class TwitterStreamProvider implements StreamsProvider, Serializable, DatumStatusCountable {
-
- public static final String STREAMS_ID = "TwitterStreamProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterStreamProvider.class);
-
- /**
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * twitter.oauth.consumerKey
- * twitter.oauth.consumerSecret
- * twitter.oauth.accessToken
- * twitter.oauth.accessTokenSecret
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterStreamProvider -Dexec.args="application.conf tweets.json"
- *
- * @param args
- */
- public static void main(String[] args) {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- TwitterStreamConfiguration config = new ComponentConfigurator<>(TwitterStreamConfiguration.class).detectConfiguration(typesafe, "twitter");
- TwitterStreamProvider provider = new TwitterStreamProvider(config);
-
- ObjectMapper mapper = StreamsJacksonMapper.getInstance(Collections.singletonList(TwitterDateTimeFormat.TWITTER_FORMAT));
-
- PrintStream outStream;
- try {
- outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- } catch (FileNotFoundException ex) {
- LOGGER.error("FileNotFoundException", ex);
- return;
- }
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- try {
- json = mapper.writeValueAsString(datum.getDocument());
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-
- private static final int MAX_BATCH = 1000;
-
- private TwitterStreamConfiguration config;
-
- public TwitterStreamConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(TwitterStreamConfiguration config) {
- this.config = config;
- }
-
- private volatile Queue<Future<List<StreamsDatum>>> providerQueue;
-
- private Authentication auth;
- protected StreamingEndpoint endpoint;
- private BasicClient client;
- private AtomicBoolean running = new AtomicBoolean(false);
- protected TwitterStreamHelper processor = new TwitterStreamHelper(this);
- private DatumStatusCounter countersCurrent = new DatumStatusCounter();
- private DatumStatusCounter countersTotal = new DatumStatusCounter();
-
- public TwitterStreamProvider() {
- this.config = new ComponentConfigurator<>(TwitterStreamConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig(), "twitter");
- }
-
- public TwitterStreamProvider(TwitterStreamConfiguration config) {
- this.config = config;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void startStream() {
- client.connect();
- running.set(true);
- }
-
- @Override
- public synchronized StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
- synchronized (this) {
- Queue<StreamsDatum> drain = new LinkedBlockingDeque<>();
- drainTo(drain);
- current = new StreamsResultSet(drain);
- current.setCounter(new DatumStatusCounter());
- current.getCounter().add(countersCurrent);
- countersTotal.add(countersCurrent);
- countersCurrent = new DatumStatusCounter();
- }
-
- return current;
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean isRunning() {
- return this.running.get() && !client.isDone();
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- Objects.requireNonNull(config.getEndpoint());
-
- Hosts hosebirdHosts;
- if (config.getEndpoint().equals("userstream") ) {
-
- hosebirdHosts = new HttpHosts(Constants.USERSTREAM_HOST);
-
- UserstreamEndpoint userstreamEndpoint = new UserstreamEndpoint();
- userstreamEndpoint.withFollowings(true);
- userstreamEndpoint.withUser(false);
- userstreamEndpoint.allReplies(false);
- endpoint = userstreamEndpoint;
- } else if (config.getEndpoint().equals("sample") ) {
-
- hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);
-
- boolean track = config.getTrack() != null && !config.getTrack().isEmpty();
- boolean follow = config.getFollow() != null && !config.getFollow().isEmpty();
-
- if ( track || follow ) {
- LOGGER.debug("***\tPRESENT\t***");
- StatusesFilterEndpoint statusesFilterEndpoint = new StatusesFilterEndpoint();
- if ( track ) {
- statusesFilterEndpoint.trackTerms(config.getTrack());
- }
- if ( follow ) {
- statusesFilterEndpoint.followings(config.getFollow());
- }
- this.endpoint = statusesFilterEndpoint;
- } else {
- endpoint = new StatusesSampleEndpoint();
- }
-
- } else if ( config.getEndpoint().endsWith("firehose")) {
- hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);
- endpoint = new StatusesFirehoseEndpoint();
- } else {
- LOGGER.error("NO ENDPOINT RESOLVED");
- return;
- }
-
- if ( config.getBasicauth() != null ) {
-
- Objects.requireNonNull(config.getBasicauth().getUsername());
- Objects.requireNonNull(config.getBasicauth().getPassword());
-
- auth = new BasicAuth(
- config.getBasicauth().getUsername(),
- config.getBasicauth().getPassword()
- );
-
- } else if ( config.getOauth() != null ) {
-
- Objects.requireNonNull(config.getOauth().getConsumerKey());
- Objects.requireNonNull(config.getOauth().getConsumerSecret());
- Objects.requireNonNull(config.getOauth().getAccessToken());
- Objects.requireNonNull(config.getOauth().getAccessTokenSecret());
-
- auth = new OAuth1(config.getOauth().getConsumerKey(),
- config.getOauth().getConsumerSecret(),
- config.getOauth().getAccessToken(),
- config.getOauth().getAccessTokenSecret());
-
- } else {
- LOGGER.error("NO AUTH RESOLVED");
- return;
- }
-
- LOGGER.debug("host={}\tendpoint={}\taut={}", hosebirdHosts, endpoint, auth);
-
- providerQueue = new LinkedBlockingQueue<>(MAX_BATCH);
-
- client = new ClientBuilder()
- .name("apache/streams/streams-contrib/streams-provider-twitter")
- .hosts(hosebirdHosts)
- .endpoint(endpoint)
- .authentication(auth)
- .connectionTimeout(1200000)
- .processor(processor)
- .build();
-
- }
-
- @Override
- public void cleanUp() {
- this.client.stop();
- this.processor.cleanUp();
- this.running.set(false);
- }
-
- @Override
- public DatumStatusCounter getDatumStatusCounter() {
- return countersTotal;
- }
-
- protected boolean addDatum(Future<List<StreamsDatum>> future) {
- try {
- ComponentUtils.offerUntilSuccess(future, providerQueue);
- countersCurrent.incrementStatus(DatumStatus.SUCCESS);
- return true;
- } catch (Exception ex) {
- countersCurrent.incrementStatus(DatumStatus.FAIL);
- LOGGER.warn("Unable to enqueue item from Twitter stream");
- return false;
- }
- }
-
- protected void drainTo(Queue<StreamsDatum> drain) {
- int count = 0;
- while (!providerQueue.isEmpty() && count <= MAX_BATCH) {
- for (StreamsDatum datum : pollForDatum()) {
- ComponentUtils.offerUntilSuccess(datum, drain);
- count++;
- }
- }
- }
-
- protected List<StreamsDatum> pollForDatum() {
- try {
- return providerQueue.poll().get();
- } catch (InterruptedException ex) {
- LOGGER.warn("Interrupted while waiting for future. Initiate shutdown.");
- this.cleanUp();
- Thread.currentThread().interrupt();
- return new ArrayList<>();
- } catch (ExecutionException ex) {
- LOGGER.warn("Error getting tweet from future");
- return new ArrayList<>();
- }
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
deleted file mode 100644
index 46d3539..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.TwitterTimelineProviderConfiguration;
-import org.apache.streams.twitter.api.StatusesUserTimelineRequest;
-import org.apache.streams.twitter.api.Twitter;
-import org.apache.streams.twitter.api.UsersLookupRequest;
-import org.apache.streams.twitter.converter.TwitterDateTimeFormat;
-import org.apache.streams.twitter.pojo.User;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static java.util.concurrent.Executors.newSingleThreadExecutor;
-
-/**
- * Retrieve recent posts from a list of user ids or names.
- */
-public class TwitterTimelineProvider implements StreamsProvider, Serializable {
-
- private static final String STREAMS_ID = "TwitterTimelineProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterTimelineProvider.class);
-
- public static final int MAX_NUMBER_WAITING = 10000;
-
- private TwitterTimelineProviderConfiguration config;
-
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- public TwitterTimelineProviderConfiguration getConfig() {
- return config;
- }
-
- protected List<String> names = new ArrayList<>();
- protected List<Long> ids = new ArrayList<>();
-
- protected volatile Queue<StreamsDatum> providerQueue;
-
- protected int idsCount;
- protected Twitter client;
-
- protected ListeningExecutorService executor;
-
- protected DateTime start;
- protected DateTime end;
-
- protected final AtomicBoolean running = new AtomicBoolean();
-
- private List<ListenableFuture<Object>> futures = new ArrayList<>();
-
- /**
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * twitter.oauth.consumerKey
- * twitter.oauth.consumerSecret
- * twitter.oauth.accessToken
- * twitter.oauth.accessTokenSecret
- * twitter.info
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterTimelineProvider -Dexec.args="application.conf tweets.json"
- *
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- TwitterTimelineProviderConfiguration config = new ComponentConfigurator<>(TwitterTimelineProviderConfiguration.class).detectConfiguration(typesafe, "twitter");
- TwitterTimelineProvider provider = new TwitterTimelineProvider(config);
-
- ObjectMapper mapper = new StreamsJacksonMapper(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList()));
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- try {
- json = mapper.writeValueAsString(datum.getDocument());
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning() );
- provider.cleanUp();
- outStream.flush();
- }
-
- public TwitterTimelineProvider(TwitterTimelineProviderConfiguration config) {
- this.config = config;
- }
-
- public Queue<StreamsDatum> getProviderQueue() {
- return this.providerQueue;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- if( configurationObject instanceof TwitterTimelineProviderConfiguration ) {
- this.config = (TwitterTimelineProviderConfiguration)configurationObject;
- }
-
- try {
- lock.writeLock().lock();
- providerQueue = constructQueue();
- } finally {
- lock.writeLock().unlock();
- }
-
- Objects.requireNonNull(providerQueue);
- Objects.requireNonNull(config.getOauth().getConsumerKey());
- Objects.requireNonNull(config.getOauth().getConsumerSecret());
- Objects.requireNonNull(config.getOauth().getAccessToken());
- Objects.requireNonNull(config.getOauth().getAccessTokenSecret());
- Objects.requireNonNull(config.getInfo());
- Objects.requireNonNull(config.getThreadsPerProvider());
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
-
- try {
- client = getTwitterClient();
- } catch (InstantiationException e) {
- LOGGER.error("InstantiationException", e);
- }
-
- Objects.requireNonNull(client);
-
- for (String s : config.getInfo()) {
- if (s != null) {
- String potentialScreenName = s.replaceAll("@", "").trim().toLowerCase();
-
- // See if it is a long, if it is, add it to the user iD list, if it is not, add it to the
- // screen name list
- try {
- ids.add(Long.parseLong(potentialScreenName));
- } catch (Exception ex) {
- names.add(potentialScreenName);
- }
- }
- }
-
- executor = MoreExecutors.listeningDecorator(
- TwitterUserInformationProvider.newFixedThreadPoolWithQueueSize(
- config.getThreadsPerProvider().intValue(),
- streamsConfiguration.getQueueSize().intValue()
- )
- );
-
- submitTimelineThreads(ids, names);
-
- }
-
- @Override
- public void startStream() {
-
- LOGGER.debug("{} startStream", STREAMS_ID);
-
- running.set(true);
-
- executor.shutdown();
-
- }
-
- protected void submitTimelineThreads(List<Long> ids, List<String> names) {
-
- for (Long id : ids) {
- StatusesUserTimelineRequest request = new StatusesUserTimelineRequest();
- request.setUserId(id);
- request.setCount(config.getPageSize());
- TwitterTimelineProviderTask providerTask = new TwitterTimelineProviderTask(
- this,
- client,
- request
- );
- ListenableFuture future = executor.submit(providerTask);
- futures.add(future);
- LOGGER.info("Thread Submitted: {}", providerTask.request);
- }
- for (String name : names) {
- StatusesUserTimelineRequest request = new StatusesUserTimelineRequest();
- request.setScreenName(name);
- request.setCount(config.getPageSize());
- TwitterTimelineProviderTask providerTask = new TwitterTimelineProviderTask(
- this,
- client,
- request
- );
- ListenableFuture future = executor.submit(providerTask);
- futures.add(future);
- LOGGER.info("Thread Submitted: {}", providerTask.request);
- }
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet result;
-
- LOGGER.debug("Providing {} docs", providerQueue.size());
-
- try {
- lock.writeLock().lock();
- result = new StreamsResultSet(providerQueue);
- result.setCounter(new DatumStatusCounter());
- providerQueue = constructQueue();
- } finally {
- lock.writeLock().unlock();
- }
-
- if ( result.size() == 0 && providerQueue.isEmpty() && executor.isTerminated() ) {
- LOGGER.info("Finished. Cleaning up...");
-
- running.set(false);
-
- LOGGER.info("Exiting");
- }
-
- return result;
-
- }
-
- protected Queue<StreamsDatum> constructQueue() {
- return new LinkedBlockingQueue<StreamsDatum>();
- }
-
- public StreamsResultSet readNew(BigInteger sequence) {
- LOGGER.debug("{} readNew", STREAMS_ID);
- throw new NotImplementedException();
- }
-
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- LOGGER.debug("{} readRange", STREAMS_ID);
- throw new NotImplementedException();
- }
-
- /**
- * get Twitter Client from TwitterUserInformationConfiguration.
- * @return result
- */
- public Twitter getTwitterClient() throws InstantiationException {
-
- return Twitter.getInstance(config);
-
- }
-
- @Override
- public void cleanUp() {
- shutdownAndAwaitTermination(executor);
- }
-
- void shutdownAndAwaitTermination(ExecutorService pool) {
- pool.shutdown(); // Disable new tasks from being submitted
- try {
- // Wait a while for existing tasks to terminate
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- pool.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- System.err.println("Pool did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- pool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
- @Override
- public boolean isRunning() {
- if (providerQueue.isEmpty() && executor.isTerminated() && Futures.allAsList(futures).isDone()) {
- LOGGER.info("Completed");
- running.set(false);
- LOGGER.info("Exiting");
- }
- return running.get();
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProviderTask.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProviderTask.java
deleted file mode 100644
index f8816e0..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProviderTask.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.api.StatusesUserTimelineRequest;
-import org.apache.streams.twitter.api.Twitter;
-import org.apache.streams.twitter.converter.TwitterDateTimeFormat;
-import org.apache.streams.twitter.pojo.Tweet;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Retrieve recent posts for a single user id.
- */
-public class TwitterTimelineProviderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterTimelineProviderTask.class);
-
- private static ObjectMapper MAPPER = new StreamsJacksonMapper(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList()));
-
- protected TwitterTimelineProvider provider;
- protected Twitter client;
- protected StatusesUserTimelineRequest request;
-
- /**
- * TwitterTimelineProviderTask constructor.
- * @param provider TwitterTimelineProvider
- * @param twitter Twitter
- * @param request StatusesUserTimelineRequest
- */
- public TwitterTimelineProviderTask(TwitterTimelineProvider provider, Twitter twitter, StatusesUserTimelineRequest request) {
- this.provider = provider;
- this.client = twitter;
- this.request = request;
- }
-
- int item_count = 0;
- int last_count = 0;
- int page_count = 0;
-
- @Override
- public void run() {
-
- LOGGER.info("Thread Starting: {}", request.toString());
-
- do {
-
- List<Tweet> statuses = client.userTimeline(request);
-
- last_count = statuses.size();
- if( statuses.size() > 0 ) {
-
- for (Tweet status : statuses) {
-
- if (item_count < provider.getConfig().getMaxItems()) {
- ComponentUtils.offerUntilSuccess(new StreamsDatum(status), provider.providerQueue);
- item_count++;
- }
-
- }
-
- Stream<Long> statusIds = statuses.stream().map(status -> status.getId());
- long minId = statusIds.reduce(Math::min).get();
- page_count++;
- request.setMaxId(minId);
-
- }
-
- }
- while (shouldContinuePulling(last_count, page_count, item_count));
-
- LOGGER.info("item_count: {} last_count: {} page_count: {} ", item_count, last_count, page_count);
-
- }
-
- public boolean shouldContinuePulling(int count, int page_count, int item_count) {
- return (
- count > 0
- && item_count < provider.getConfig().getMaxItems()
- && page_count <= provider.getConfig().getMaxPages());
- }
-
-
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterUserInformationProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterUserInformationProvider.java
deleted file mode 100644
index 2ddfd08..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterUserInformationProvider.java
+++ /dev/null
@@ -1,406 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.TwitterFollowingConfiguration;
-import org.apache.streams.twitter.TwitterUserInformationConfiguration;
-import org.apache.streams.twitter.api.Twitter;
-import org.apache.streams.twitter.api.UsersLookupRequest;
-import org.apache.streams.twitter.converter.TwitterDateTimeFormat;
-import org.apache.streams.twitter.pojo.User;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-import org.apache.commons.lang.NotImplementedException;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import static java.util.concurrent.Executors.newSingleThreadExecutor;
-
-/**
- * Retrieve current profile status from a list of user ids or names.
- */
-public class TwitterUserInformationProvider implements StreamsProvider, Serializable {
-
- private static final String STREAMS_ID = "TwitterUserInformationProvider";
-
- private static ObjectMapper MAPPER = new StreamsJacksonMapper(Collections.singletonList(TwitterDateTimeFormat.TWITTER_FORMAT));
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterUserInformationProvider.class);
-
- public static final int MAX_NUMBER_WAITING = 1000;
-
- private TwitterUserInformationConfiguration config;
-
- protected List<String> names = new ArrayList<>();
- protected List<Long> ids = new ArrayList<>();
-
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- protected volatile Queue<StreamsDatum> providerQueue;
-
- public TwitterUserInformationConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(TwitterUserInformationConfiguration config) {
- this.config = config;
- }
-
- protected Twitter client;
-
- protected ListeningExecutorService executor;
-
- protected DateTime start;
- protected DateTime end;
-
- protected final AtomicBoolean running = new AtomicBoolean();
-
- private List<ListenableFuture<Object>> futures = new ArrayList<>();
-
- /**
- * To use from command line:
- *
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- *
- * <p/>
- * twitter.oauth.consumerKey
- * twitter.oauth.consumerSecret
- * twitter.oauth.accessToken
- * twitter.oauth.accessTokenSecret
- * twitter.info
- *
- * <p/>
- * Launch using:
- *
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.twitter.provider.TwitterUserInformationProvider -Dexec.args="application.conf tweets.json"
- *
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- TwitterUserInformationConfiguration config = new ComponentConfigurator<>(TwitterUserInformationConfiguration.class).detectConfiguration(typesafe, "twitter");
- TwitterUserInformationProvider provider = new TwitterUserInformationProvider(config);
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- try {
- json = MAPPER.writeValueAsString(datum.getDocument());
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while ( provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-
- // TODO: this should be abstracted out
- public static ExecutorService newFixedThreadPoolWithQueueSize(int numThreads, int queueSize) {
- return new ThreadPoolExecutor(numThreads, numThreads,
- 5000L, TimeUnit.MILLISECONDS,
- new ArrayBlockingQueue<>(queueSize, true), new ThreadPoolExecutor.CallerRunsPolicy());
- }
-
- /**
- * TwitterUserInformationProvider constructor.
- * Resolves config from JVM properties 'twitter'.
- */
- public TwitterUserInformationProvider() {
- this.config = new ComponentConfigurator<>(TwitterUserInformationConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig().getConfig("twitter"));
- }
-
- public TwitterUserInformationProvider(TwitterUserInformationConfiguration config) {
- this.config = config;
- }
-
- public Queue<StreamsDatum> getProviderQueue() {
- return this.providerQueue;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- if ( configurationObject instanceof TwitterFollowingConfiguration ) {
- config = (TwitterUserInformationConfiguration) configurationObject;
- }
-
- Objects.requireNonNull(config);
- Objects.requireNonNull(config.getOauth());
- Objects.requireNonNull(config.getOauth().getConsumerKey());
- Objects.requireNonNull(config.getOauth().getConsumerSecret());
- Objects.requireNonNull(config.getOauth().getAccessToken());
- Objects.requireNonNull(config.getOauth().getAccessTokenSecret());
- Objects.requireNonNull(config.getInfo());
- Objects.requireNonNull(config.getThreadsPerProvider());
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
-
- Objects.requireNonNull(streamsConfiguration.getQueueSize());
-
- try {
- client = getTwitterClient();
- } catch (InstantiationException e) {
- LOGGER.error("InstantiationException", e);
- }
-
- Objects.requireNonNull(client);
-
- try {
- lock.writeLock().lock();
- providerQueue = constructQueue();
- } finally {
- lock.writeLock().unlock();
- }
-
- Objects.requireNonNull(providerQueue);
-
- for (String s : config.getInfo()) {
- if (s != null) {
- String potentialScreenName = s.replaceAll("@", "").trim().toLowerCase();
-
- // See if it is a long, if it is, add it to the user iD list, if it is not, add it to the
- // screen name list
- try {
- ids.add(Long.parseLong(potentialScreenName));
- } catch (Exception ex) {
- names.add(potentialScreenName);
- }
- }
- }
-
- executor = MoreExecutors.listeningDecorator(
- TwitterUserInformationProvider.newFixedThreadPoolWithQueueSize(
- config.getThreadsPerProvider().intValue(),
- streamsConfiguration.getQueueSize().intValue()
- )
- );
-
- Objects.requireNonNull(executor);
-
- // Twitter allows for batches up to 100 per request, but you cannot mix types
- submitUserInformationThreads(ids, names);
- }
-
- protected void submitUserInformationThreads(List<Long> ids, List<String> names) {
-
- int idsIndex = 0;
- while( idsIndex + 100 < ids.size() ) {
- List<Long> batchIds = ids.subList(idsIndex, idsIndex + 100);
- TwitterUserInformationProviderTask providerTask = new TwitterUserInformationProviderTask(
- this,
- client,
- new UsersLookupRequest().withUserId(batchIds));
- ListenableFuture future = executor.submit(providerTask);
- futures.add(future);
- LOGGER.info("Thread Submitted: {}", providerTask.request);
- idsIndex += 100;
- }
- if (ids.size() >= idsIndex) {
- List<Long> batchIds = ids.subList(idsIndex, ids.size());
- TwitterUserInformationProviderTask providerTask = new TwitterUserInformationProviderTask(
- this,
- client,
- new UsersLookupRequest().withUserId(batchIds));
- ListenableFuture future = executor.submit(providerTask);
- futures.add(future);
- LOGGER.info("Thread Submitted: {}", providerTask.request);
- }
-
- int namesIndex = 0;
- while( namesIndex + 100 < names.size() ) {
- List<String> batchNames = names.subList(namesIndex, namesIndex + 100);
- TwitterUserInformationProviderTask providerTask = new TwitterUserInformationProviderTask(
- this,
- client,
- new UsersLookupRequest().withScreenName(batchNames));
- ListenableFuture future = executor.submit(providerTask);
- futures.add(future);
- LOGGER.info("Thread Submitted: {}", providerTask.request);
- namesIndex += 100;
- }
- if (names.size() >= namesIndex) {
- List<String> batchNames = names.subList(namesIndex, names.size());
- TwitterUserInformationProviderTask providerTask = new TwitterUserInformationProviderTask(
- this,
- client,
- new UsersLookupRequest().withScreenName(batchNames));
- ListenableFuture future = executor.submit(providerTask);
- futures.add(future);
- LOGGER.info("Thread Submitted: {}", providerTask.request);
- }
-
- }
-
- @Override
- public void startStream() {
-
- Objects.requireNonNull(executor);
-
- LOGGER.info("startStream: {} Threads", futures.size());
-
- running.set(true);
-
- executor.shutdown();
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet result;
-
- try {
- lock.writeLock().lock();
- result = new StreamsResultSet(providerQueue);
- result.setCounter(new DatumStatusCounter());
- providerQueue = constructQueue();
- LOGGER.debug("readCurrent: {} Documents", result.size());
- } finally {
- lock.writeLock().unlock();
- }
-
- return result;
-
- }
-
- protected Queue<StreamsDatum> constructQueue() {
- return new LinkedBlockingQueue<>();
- }
-
- public StreamsResultSet readNew(BigInteger sequence) {
- LOGGER.debug("{} readNew", STREAMS_ID);
- throw new NotImplementedException();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- LOGGER.debug("{} readRange", STREAMS_ID);
- this.start = start;
- this.end = end;
- readCurrent();
- return (StreamsResultSet)providerQueue.iterator();
- }
-
-
- @Override
- public boolean isRunning() {
- if ( providerQueue.isEmpty() && executor.isTerminated() && Futures.allAsList(futures).isDone() ) {
- LOGGER.info("All Threads Completed");
- running.set(false);
- LOGGER.info("Exiting");
- }
- return running.get();
- }
-
- void shutdownAndAwaitTermination(ExecutorService pool) {
- pool.shutdown(); // Disable new tasks from being submitted
- try {
- // Wait a while for existing tasks to terminate
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- pool.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- System.err.println("Pool did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- pool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
- protected Twitter getTwitterClient() throws InstantiationException {
- return Twitter.getInstance(config);
- }
-
- @Override
- public void cleanUp() {
- shutdownAndAwaitTermination(executor);
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterUserInformationProviderTask.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterUserInformationProviderTask.java
deleted file mode 100644
index 5dbb784..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterUserInformationProviderTask.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.api.Twitter;
-import org.apache.streams.twitter.api.UsersLookupRequest;
-import org.apache.streams.twitter.converter.TwitterDateTimeFormat;
-import org.apache.streams.twitter.pojo.User;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Retrieve recent posts for a single user id.
- */
-public class TwitterUserInformationProviderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterUserInformationProviderTask.class);
-
- private static ObjectMapper MAPPER = new StreamsJacksonMapper(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList()));
-
- protected TwitterUserInformationProvider provider;
- protected Twitter client;
- protected UsersLookupRequest request;
-
- /**
- * TwitterTimelineProviderTask constructor.
- * @param provider TwitterUserInformationProvider
- * @param twitter Twitter
- * @param request UsersLookupRequest
- */
- public TwitterUserInformationProviderTask(TwitterUserInformationProvider provider, Twitter twitter, UsersLookupRequest request) {
- this.provider = provider;
- this.client = twitter;
- this.request = request;
- }
-
- @Override
- public void run() {
-
- LOGGER.info("Thread Starting: {}", request.toString());
-
- List<User> users = client.lookup(request);
-
- for (User user : users) {
- ComponentUtils.offerUntilSuccess(new StreamsDatum(user), provider.providerQueue);
- }
-
- LOGGER.info("Thread Finished: {}", request.toString());
-
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/AccountSettingsResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/AccountSettingsResponse.json
deleted file mode 100644
index 40b5747..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/AccountSettingsResponse.json
+++ /dev/null
@@ -1,108 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.AccountSessionsResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/webhooks/reference/get/account_activity/webhooks",
- "properties": {
- "always_use_https": {
- "type": "boolean"
- },
- "discoverable_by_email": {
- "type": "boolean"
- },
- "geo_enabled": {
- "type": "boolean"
- },
- "language": {
- "type": "string"
- },
- "protected": {
- "type": "boolean"
- },
- "screen_name": {
- "type": "string"
- },
- "show_all_inline_media": {
- "type": "boolean"
- },
- "sleep_time": {
- "type": "object",
- "properties": {
- "enabled": {
- "type": "boolean"
- },
- "end_time": {
- "type": "string",
- "format": "date-time"
- },
- "start_time": {
- "type": "string",
- "format": "date-time"
- }
- }
- },
- "time_zone": {
- "type": "object",
- "properties": {
- "name": {
- "type": "string"
- },
- "tzinfo_name": {
- "type": "string"
- },
- "utc_offset": {
- "type": "integer"
- }
- }
- },
- "trend_location": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "country": {
- "type": "string"
- },
- "countryCode": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "parentid": {
- "type": "integer"
- },
- "placeType": {
- "type": "object",
- "properties": {
- "code": {
- "type": "integer"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "url": {
- "type": "string"
- },
- "woeid": {
- "type": "integer"
- }
- }
- }
- },
- "use_cookie_personalization": {
- "type": "boolean"
- },
- "allow_contributor_request": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/EventShowResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/EventShowResponse.json
deleted file mode 100644
index 95fc31b..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/EventShowResponse.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.EventShowResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/webhooks/reference/get/account_activity/webhooks",
- "properties": {
- "event": {
- "type": {
- "$ref": "../pojo/DirectMessageEvent.json"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/EventsListRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/EventsListRequest.json
deleted file mode 100644
index f8b368a..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/EventsListRequest.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.EventsListRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/webhooks/reference/get/account_activity/webhooks",
- "properties": {
- "count": {
- "description": "Max number of events to be returned. 20 default. 50 max.",
- "type": "integer",
- "required": false
- },
- "cursor": {
- "description": "For paging through result sets greater than 1 page, use the “next_cursor” property from the previous request.",
- "type": "integer",
- "required": false
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/EventsListResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/EventsListResponse.json
deleted file mode 100644
index 66c4564..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/EventsListResponse.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.EventsListResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/webhooks/reference/get/account_activity/webhooks",
- "properties": {
- "events": {
- "type": "array",
- "items": {
- "type": {
- "$ref": "../pojo/DirectMessageEvent.json"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FavoritesListRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FavoritesListRequest.json
deleted file mode 100644
index d6e0c84..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FavoritesListRequest.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.FavoritesListRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "user_id": {
- "description": "The ID of the user for whom to return results. Either an id or screen_name is required for this method.",
- "required": false,
- "type": "integer"
- },
- "screen_name": {
- "description": "The screen name of the user for whom to return results. Either a id or screen_name is required for this method.",
- "required": false,
- "type": "string"
- },
- "count": {
- "description": "Specifies the number of records to retrieve. Must be less than or equal to 200; defaults to 20. The value of count is best thought of as a limit to the number of tweets to return because suspended or deleted content is removed after the count has been applied.",
- "required": false,
- "type": "integer"
- },
- "since_id": {
- "description": "Returns results with an ID greater than (that is, more recent than) the specified ID. There are limits to the number of Tweets which can be accessed through the API. If the limit of Tweets has occured since the since_id, the since_id will be forced to the oldest ID available.",
- "required": false,
- "type": "integer"
- },
- "max_id": {
- "description": "Returns results with an ID less than (that is, older than) or equal to the specified ID.",
- "required": false,
- "type": "integer"
- },
- "include_entities": {
- "description": "The entities node will be omitted when set to false .",
- "required": false,
- "type": "boolean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowersIdsRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowersIdsRequest.json
deleted file mode 100644
index 2b98689..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowersIdsRequest.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.api.FollowersIdsRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://dev.twitter.com/rest/reference/get/followers/ids",
- "extends": { "$ref": "FollowingIdsRequest.json" },
- "properties": {}
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowersIdsResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowersIdsResponse.json
deleted file mode 100644
index 03dd83c..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowersIdsResponse.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.FollowersIdsResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/get/followers/ids",
- "properties": {
- "ids": {
- "type": "array",
- "items": {
- "type": "integer"
- }
- },
- "previous_cursor": {
- "type": "integer"
- },
- "previous_cursor_str": {
- "type": "string"
- },
- "next_cursor": {
- "type": "integer"
- },
- "next_cursor_str": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowersListRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowersListRequest.json
deleted file mode 100644
index c588579..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowersListRequest.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.api.FollowersListRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://dev.twitter.com/rest/reference/get/followers/list",
- "extends": { "$ref": "FollowingListRequest.json" },
- "properties": {}
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowersListResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowersListResponse.json
deleted file mode 100644
index e8e46a1..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowersListResponse.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.FollowersListResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/get/followers/list",
- "properties": {
- "users": {
- "type": "array",
- "items": {
- "type": "object",
- "$ref": "../pojo/User.json"
- }
- },
- "previous_cursor": {
- "type": "integer"
- },
- "previous_cursor_str": {
- "type": "string"
- },
- "next_cursor": {
- "type": "integer"
- },
- "next_cursor_str": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowingIdsRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowingIdsRequest.json
deleted file mode 100644
index 81804fa..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowingIdsRequest.json
+++ /dev/null
@@ -1,36 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.FollowingIdsRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "description": "The ID of the user for whom to return results for.",
- "required": false,
- "type": "integer"
- },
- "screen_name": {
- "description": "The screen name of the user for whom to return results for.",
- "required": false,
- "type": "string"
- },
- "cursor": {
- "description": "Causes the list of connections to be broken into pages of no more than 5000 IDs at a time. The number of IDs returned is not guaranteed to be 5000 as suspended users are filtered out after connections are queried. If no cursor is provided, a value of -1 will be assumed, which is the first page.\nThe response from the API will include a previous_cursor and next_cursor to allow paging back and forth.",
- "required": false,
- "type": "integer"
- },
- "stringify_ids": {
- "description": "Many programming environments will not consume our Tweet ids due to their size. Provide this option to have ids returned as strings instead.",
- "required": false,
- "type": "boolean"
- },
- "count": {
- "description": "Specifies the number of IDs attempt retrieval of, up to a maximum of 5,000 per distinct request. The value of count is best thought of as a limit to the number of results to return. When using the count parameter with this method, it is wise to use a consistent count value across all requests to the same user’s collection. Usage of this parameter is encouraged in environments where all 5,000 IDs constitutes too large of a response.",
- "required": false,
- "type": "integer"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowingListRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowingListRequest.json
deleted file mode 100644
index 90295b5..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FollowingListRequest.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.FollowingListRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "description": "The ID of the user for whom to return results for.",
- "required": false,
- "type": "integer"
- },
- "screen_name": {
- "description": "The screen name of the user for whom to return results for.",
- "required": false,
- "type": "string"
- },
- "cursor": {
- "description": "Causes the list of connections to be broken into pages of no more than 5000 IDs at a time. The number of IDs returned is not guaranteed to be 5000 as suspended users are filtered out after connections are queried. If no cursor is provided, a value of -1 will be assumed, which is the first page.\nThe response from the API will include a previous_cursor and next_cursor to allow paging back and forth.",
- "required": false,
- "type": "integer"
- },
- "count": {
- "description": "The number of users to return per page, up to a maximum of 200. Defaults to 20.",
- "required": false,
- "type": "integer"
- },
- "skip_status": {
- "description": "When set to either true , t or 1 , statuses will not be included in the returned user objects. If set to any other value, statuses will be included.",
- "required": false,
- "type": "boolean"
- },
- "include_user_entities": {
- "description": "The user object entities node will not be included when set to false.",
- "required": false,
- "type": "integer"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendsIdsRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendsIdsRequest.json
deleted file mode 100644
index fda54f8..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendsIdsRequest.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.api.FriendsIdsRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://dev.twitter.com/rest/reference/get/friends/ids",
- "extends": { "$ref": "FollowingIdsRequest.json" },
- "properties": {}
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendsIdsResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendsIdsResponse.json
deleted file mode 100644
index fc46841..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendsIdsResponse.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.FriendsIdsResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/get/friends/ids",
- "properties": {
- "ids": {
- "type": "array",
- "items": {
- "type": "integer"
- }
- },
- "previous_cursor": {
- "type": "integer"
- },
- "previous_cursor_str": {
- "type": "string"
- },
- "next_cursor": {
- "type": "integer"
- },
- "next_cursor_str": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendsListRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendsListRequest.json
deleted file mode 100644
index 75bba94..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendsListRequest.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.api.FriendsListRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://dev.twitter.com/rest/reference/get/friends/list",
- "extends": { "$ref": "FollowingListRequest.json" },
- "properties": {}
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendsListResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendsListResponse.json
deleted file mode 100644
index ac357aa..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendsListResponse.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.FriendsListResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/get/friends/list",
- "properties": {
- "users": {
- "type": "array",
- "items": {
- "type": "object",
- "$ref": "../pojo/User.json"
- }
- },
- "previous_cursor": {
- "type": "integer"
- },
- "previous_cursor_str": {
- "type": "string"
- },
- "next_cursor": {
- "type": "integer"
- },
- "next_cursor_str": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipCreateRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipCreateRequest.json
deleted file mode 100644
index 8e9d729..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipCreateRequest.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.api.FriendshipCreateRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://https://dev.twitter.com/rest/reference/post/friendships/create",
- "properties": {
- "follow": {
- "description": "Enable notifications for the target user.",
- "required": false,
- "type": "boolean"
- },
- "screen_name": {
- "description": "The screen name of the user for whom to befriend.",
- "required": false,
- "type": "string"
- },
- "user_id": {
- "description": "The ID of the user for whom to befriend.",
- "required": false,
- "type": "integer"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipDestroyRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipDestroyRequest.json
deleted file mode 100644
index cda5b20..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipDestroyRequest.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.api.FriendshipDestroyRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://https://dev.twitter.com/rest/reference/post/friendships/destroy",
- "properties": {
- "screen_name": {
- "description": "The screen name of the user for whom to befriend.",
- "required": false,
- "type": "string"
- },
- "user_id": {
- "description": "The ID of the user for whom to befriend.",
- "required": false,
- "type": "integer"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipShowRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipShowRequest.json
deleted file mode 100644
index e26402c..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipShowRequest.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.api.FriendshipShowResponse",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://https://dev.twitter.com/rest/reference/get/friendships/show",
- "properties": {
- "relationship": {
- "$ref": "../pojo/Relationship.json"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipShowResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipShowResponse.json
deleted file mode 100644
index 966fe2c..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipShowResponse.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.api.FriendshipShowRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://https://dev.twitter.com/rest/reference/get/friendships/show",
- "properties": {
- "source_id": {
- "description": "The user_id of the subject user.",
- "required": false,
- "type": "integer"
- },
- "source_screen_name": {
- "description": "The screen_name of the subject user.",
- "required": false,
- "type": "string"
- },
- "target_id": {
- "description": "The user_id of the target user.",
- "required": false,
- "type": "integer"
- },
- "target_screen_name": {
- "description": "The screen_name of the target user.",
- "required": false,
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipUpdateRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipUpdateRequest.json
deleted file mode 100644
index 6525a7e..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipUpdateRequest.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.api.FriendshipUpdateRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://https://dev.twitter.com/rest/reference/post/friendships/update",
- "properties": {
- "device": {
- "description": "Enable/disable device notifications from the target user.",
- "required": false,
- "type": "boolean"
- },
- "retweets": {
- "description": "Enable/disable retweets from the target user.",
- "required": false,
- "type": "boolean"
- },
- "screen_name": {
- "description": "The screen name of the user for whom to befriend.",
- "required": false,
- "type": "string"
- },
- "user_id": {
- "description": "The ID of the user for whom to befriend.",
- "required": false,
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipsLookupRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipsLookupRequest.json
deleted file mode 100644
index f7d9ed7..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipsLookupRequest.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.FriendshipsLookupRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/get/friendships/lookup",
- "properties": {
- "user_id": {
- "description": "A comma separated list of user IDs, up to 100 are allowed in a single request. You are strongly encouraged to use a POST for larger requests.",
- "required": true,
- "type": "array",
- "items": {
- "type": "integer"
- }
- },
- "screen_name": {
- "description": "A comma separated list of screen names, up to 100 are allowed in a single request. You are strongly encouraged to use a POST for larger (up to 100 screen names) requests.",
- "required": true,
- "type": "array",
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipsOutgoingRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipsOutgoingRequest.json
deleted file mode 100644
index 2adf165..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/FriendshipsOutgoingRequest.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.api.FriendshipsOutgoingRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://https://dev.twitter.com/rest/reference/get/friendships/outgoing",
- "properties": {
- "cursor": {
- "description": "Causes the list of connections to be broken into pages of no more than 5000 IDs at a time. The number of IDs returned is not guaranteed to be 5000 as suspended users are filtered out after connections are queried. If no cursor is provided, a value of -1 will be assumed, which is the first page.",
- "required": false,
- "type": "integer"
- },
- "stringify_ids": {
- "description": "Many programming environments will not consume our Tweet ids due to their size. Provide this option to have ids returned as strings instead.",
- "required": false,
- "type": "boolean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/GeoSearchRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/GeoSearchRequest.json
deleted file mode 100644
index 6000327..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/GeoSearchRequest.json
+++ /dev/null
@@ -1,52 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.GeoSearchRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://dev.twitter.com/rest/reference/get/users/show/id",
- "properties": {
- "lat": {
- "description": "The latitude to search around. This parameter will be ignored unless it is inside the range -90.0 to +90.0 (North is positive) inclusive. It will also be ignored if there isn’t a corresponding long parameter.",
- "required": false,
- "type": "number"
- },
- "long": {
- "description": "The longitude to search around. The valid ranges for longitude is -180.0 to +180.0 (East is positive) inclusive. This parameter will be ignored if outside that range, if it is not a number, if geo_enabled is disabled, or if there not a corresponding lat parameter.",
- "required": false,
- "type": "number"
- },
- "query": {
- "description": "Free-form text to match against while executing a geo-based query, best suited for finding nearby locations by name. Remember to URL encode the query.",
- "required": false,
- "type": "string"
- },
- "ip": {
- "description": "An IP address. Used when attempting to fix geolocation based off of the user’s IP address.",
- "required": false,
- "type": "string"
- },
- "granularity": {
- "description": "This is the minimal granularity of place types to return and must be one of: poi , neighborhood , city , admin or country . If no granularity is provided for the request neighborhood is assumed. Setting this to city , for example, will find places which have a type of city , admin or country .",
- "required": false,
- "type": "string"
- },
- "accuracy": {
- "description": "A hint on the region in which to search. If a number, then this is a radius in meters, but it can also take a string that is suffixed with ft to specify feet. If this is not passed in, then it is assumed to be 0m . If coming from a device, in practice, this value is whatever accuracy the device has measuring its location (whether it be coming from a GPS, WiFi triangulation, etc.).",
- "required": false,
- "type": "boolean"
- },
- "max_results": {
- "description": "A hint as to the number of results to return. This does not guarantee that the number of results returned will equal max_results, but instead informs how many nearby results to return. Ideally, only pass in the number of places you intend to display to the user here.",
- "required": false,
- "type": "integer"
- },
- "contained_within": {
- "description": "This is the place_id which you would like to restrict the search results to. Setting this value means only places within the given place_id will be found. Specify a place_id.",
- "required": false,
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/GeoSearchResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/GeoSearchResponse.json
deleted file mode 100644
index 33acc98..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/GeoSearchResponse.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.GeoSearchResponse",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://dev.twitter.com/rest/reference/get/users/show/id",
- "properties": {
- "query": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.api.GeoSearchResponseQuery",
- "properties": {
- "params": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.api.GeoSearchResponseQueryParams",
- "properties": {
- "accuracy": {
- "type": "number"
- },
- "autocomplete": {
- "type": "boolean"
- },
- "granularity": {
- "type": "string"
- },
- "query": {
- "type": "string"
- },
- "trim_place": {
- "type": "boolean"
- }
- }
- },
- "type": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- },
- "result": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.api.GeoSearchResponseResult",
- "properties": {
- "places": {
- "type": "array",
- "items": {
- "$ref": "../pojo/Place.json"
- }
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/MediaStatusRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/MediaStatusRequest.json
deleted file mode 100644
index c4cf460..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/MediaStatusRequest.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.MediaStatusRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "command": {
- "description": "Must be set to STATUS (case sensitive).",
- "required": true,
- "type": "string",
- "enum": [
- "STATUS"
- ]
- },
- "media_id": {
- "description": "The media_id returned from the INIT command.",
- "required": true,
- "type": "integer"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/MediaStatusResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/MediaStatusResponse.json
deleted file mode 100644
index 058ff5e..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/MediaStatusResponse.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.MediaStatusResponse",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "media_id": {
- "description": "The media_id returned from the INIT command.",
- "required": true,
- "type": "integer"
- },
- "media_id_string": {
- "description": "The media_id returned from the INIT command.",
- "required": true,
- "type": "integer"
- },
- "expires_after_seconds": {
- "required": false,
- "type": "integer"
- },
- "processing_info": {
- "description": "he response body contains processing_info field which provides information about current state of media processing operation.",
- "required": true,
- "type": "object",
- "properties": {
- "state": {
- "description": "It contains a state field which has transition flow: “pending” -> “in_progress” -> [“failed” | “succeeded”]. You can not use the media_id to create Tweet or other entities before the state field is set to “succeeded”.",
- "type": "string",
- "enum": [
- "pending",
- "in_progress",
- "failed",
- "succeeded"
- ]
- },
- "progress_percent": {
- "description": "Optional [0-100] int value. Please don't use it as a replacement of \"state\" field.",
- "required": false,
- "type": "integer"
- },
- "check_after_secs": {
- "required": false,
- "type": "integer"
- }
- }
- },
- "media_id": {
- "description": "The media_id returned from the INIT command.",
- "required": true,
- "type": "integer"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/MessageCreateRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/MessageCreateRequest.json
deleted file mode 100644
index 4b23a97..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/MessageCreateRequest.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.MessageCreateRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/post/direct_messages/events/new",
- "properties": {
- "event": {
- "type": {
- "$ref": "../pojo/DirectMessageEvent.json"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/ResponseErrors.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/ResponseErrors.json
deleted file mode 100644
index 363ac36..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/ResponseErrors.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.ResponseErrors",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "errors": {
- "description": "When the Twitter API returns error messages, it does so in JSON format.",
- "required": false,
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "code": {
- "type": "integer"
- },
- "message": {
- "type": "string"
- }
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesHomeTimelineRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesHomeTimelineRequest.json
deleted file mode 100644
index c92e33f..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesHomeTimelineRequest.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.StatusesHomeTimelineRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://api.twitter.com/1.1/statuses/home_timeline.json",
- "properties": {
- "count": {
- "description": "Specifies the number of records to retrieve. Must be less than or equal to 200. Defaults to 20. The value of count is best thought of as a limit to the number of tweets to return because suspended or deleted content is removed after the count has been applied.",
- "required": false,
- "type": "integer"
- },
- "since_id": {
- "description": "Returns results with an ID greater than (that is, more recent than) the specified ID. There are limits to the number of Tweets which can be accessed through the API. If the limit of Tweets has occured since the since_id, the since_id will be forced to the oldest ID available.",
- "required": false,
- "type": "integer"
- },
- "max_id": {
- "description": "Returns results with an ID less than (that is, older than) or equal to the specified ID.",
- "required": false,
- "type": "integer"
- },
- "trim_user": {
- "description": "When set to either true , t or 1 , each Tweet returned in a timeline will include a user object including only the status authors numerical ID. Omit this parameter to receive the complete user object.",
- "required": false,
- "type": "boolean"
- },
- "exclude_replies": {
- "description": "This parameter will prevent replies from appearing in the returned timeline. Using exclude_replies with the count parameter will mean you will receive up-to count tweets — this is because the count parameter retrieves that many Tweets before filtering out retweets and replies. This parameter is only supported for JSON and XML responses.",
- "required": false,
- "type": "boolean"
- },
- "include_entities": {
- "description": "The entities node will not be included when set to false.",
- "required": false,
- "type": "boolean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesLookupRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesLookupRequest.json
deleted file mode 100644
index 4306b6b..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesLookupRequest.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.StatusesLookupRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://dev.twitter.com/rest/reference/get/statuses/lookup",
- "properties": {
- "id": {
- "description": "A comma separated list of Tweet IDs, up to 100 are allowed in a single request.",
- "required": true,
- "type": "array",
- "items": {
- "type": "integer"
- }
- },
- "trim_user": {
- "description": "When set to either true , t or 1 , each Tweet returned in a timeline will include a user object including only the status authors numerical ID. Omit this parameter to receive the complete user object.",
- "required": false,
- "type": "boolean"
- },
- "include_entities": {
- "description": "The entities node will not be included when set to false.",
- "required": false,
- "type": "boolean"
- },
- "map": {
- "description": "When using the map parameter, Tweets that do not exist or cannot be viewed by the current user will still have their key represented but with an explicitly null value paired with it.",
- "required": false,
- "type": "boolean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesMentionsTimelineRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesMentionsTimelineRequest.json
deleted file mode 100644
index 2ccb65a..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesMentionsTimelineRequest.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.StatusesMentionsTimelineRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://api.twitter.com/1.1/statuses/mentions_timeline.json",
- "properties": {
- "count": {
- "description": "Specifies the number of records to retrieve. Must be less than or equal to 200. Defaults to 20. The value of count is best thought of as a limit to the number of tweets to return because suspended or deleted content is removed after the count has been applied.",
- "required": false,
- "type": "integer"
- },
- "since_id": {
- "description": "Returns results with an ID greater than (that is, more recent than) the specified ID. There are limits to the number of Tweets which can be accessed through the API. If the limit of Tweets has occured since the since_id, the since_id will be forced to the oldest ID available.",
- "required": false,
- "type": "integer"
- },
- "max_id": {
- "description": "Returns results with an ID less than (that is, older than) or equal to the specified ID.",
- "required": false,
- "type": "integer"
- },
- "trim_user": {
- "description": "When set to either true , t or 1 , each Tweet returned in a timeline will include a user object including only the status authors numerical ID. Omit this parameter to receive the complete user object.",
- "required": false,
- "type": "boolean"
- },
- "include_entities": {
- "description": "The entities node will not be included when set to false.",
- "required": false,
- "type": "boolean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesShowRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesShowRequest.json
deleted file mode 100644
index 45278d9..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesShowRequest.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.StatusesShowRequest",
- "javaInterfaces": [
- "java.io.Serializable",
- "org.apache.streams.twitter.api.Statuses.StatusesShowRequestAnnotations"
- ],
- "description": "https://dev.twitter.com/rest/reference/get/statuses/show/id",
- "properties": {
- "id": {
- "description": "The numerical ID of the desired Tweet.",
- "required": true,
- "type": "integer"
- },
- "trim_user": {
- "description": "When set to either true , t or 1 , each Tweet returned in a timeline will include a user object including only the status authors numerical ID. Omit this parameter to receive the complete user object.",
- "required": false,
- "type": "boolean"
- },
- "include_my_retweet": {
- "description": "When set to either true , t or 1 , any Tweets returned that have been retweeted by the authenticating user will include an additional current_user_retweet node, containing the ID of the source status for the retweet.",
- "required": false,
- "type": "boolean"
- },
- "include_entities": {
- "description": "The entities node will not be included when set to false.",
- "required": false,
- "type": "boolean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesUserTimelineRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesUserTimelineRequest.json
deleted file mode 100644
index bdb41df..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/StatusesUserTimelineRequest.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.StatusesUserTimelineRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://api.twitter.com/1.1/statuses/user_timeline.json",
- "properties": {
- "user_id": {
- "description": "The ID of the user for whom to return results for.",
- "required": false,
- "type": "integer"
- },
- "screen_name": {
- "description": "The screen name of the user for whom to return results for.",
- "required": false,
- "type": "string"
- },
- "count": {
- "description": "Specifies the number of Tweets to try and retrieve, up to a maximum of 200 per distinct request. The value of count is best thought of as a limit to the number of Tweets to return because suspended or deleted content is removed after the count has been applied. We include retweets in the count, even if include_rts is not supplied. It is recommended you always send include_rts=1 when using this API method.",
- "required": false,
- "type": "integer"
- },
- "since_id": {
- "description": "Returns results with an ID greater than (that is, more recent than) the specified ID. There are limits to the number of Tweets which can be accessed through the API. If the limit of Tweets has occured since the since_id, the since_id will be forced to the oldest ID available.",
- "required": false,
- "type": "integer"
- },
- "max_id": {
- "description": "Returns results with an ID less than (that is, older than) or equal to the specified ID.",
- "required": false,
- "type": "integer"
- },
- "trim_user": {
- "description": "When set to either true , t or 1 , each Tweet returned in a timeline will include a user object including only the status authors numerical ID. Omit this parameter to receive the complete user object.",
- "required": false,
- "type": "boolean"
- },
- "exclude_replies": {
- "description": "This parameter will prevent replies from appearing in the returned timeline. Using exclude_replies with the count parameter will mean you will receive up-to count tweets — this is because the count parameter retrieves that many Tweets before filtering out retweets and replies. This parameter is only supported for JSON and XML responses.",
- "required": false,
- "type": "boolean"
- },
- "contributor_details": {
- "description": "This parameter enhances the contributors element of the status response to include the screen_name of the contributor. By default only the user_id of the contributor is included.",
- "required": false,
- "type": "boolean"
- },
- "include_rts": {
- "description": "When set to false , the timeline will strip any native retweets (though they will still count toward both the maximal length of the timeline and the slice selected by the count parameter). Note: If you’re using the trim_user parameter in conjunction with include_rts, the retweets will still contain a full user object.",
- "required": false,
- "type": "boolean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/UsersLookupRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/UsersLookupRequest.json
deleted file mode 100644
index 3f37ae5..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/UsersLookupRequest.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.UsersLookupRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/get/users/lookup",
- "properties": {
- "user_id": {
- "description": "A comma separated list of user IDs, up to 100 are allowed in a single request. You are strongly encouraged to use a POST for larger requests.",
- "required": true,
- "type": "array",
- "items": {
- "type": "integer"
- }
- },
- "screen_name": {
- "description": "A comma separated list of screen names, up to 100 are allowed in a single request. You are strongly encouraged to use a POST for larger (up to 100 screen names) requests.",
- "required": true,
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "include_entities": {
- "description": "The entities node will not be included when set to false.",
- "required": false,
- "type": "boolean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/UsersShowRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/UsersShowRequest.json
deleted file mode 100644
index c7d04ce..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/UsersShowRequest.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.api.UsersShowRequest",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "https://dev.twitter.com/rest/reference/get/users/show/id",
- "properties": {
- "user_id": {
- "description": "The ID of the user for whom to return results. Either an id or screen_name is required for this method.",
- "required": false,
- "type": "integer"
- },
- "screen_name": {
- "description": "The screen name of the user for whom to return results. Either a id or screen_name is required for this method.",
- "required": false,
- "type": "string"
- },
- "include_entities": {
- "description": "The entities node will not be included when set to false.",
- "required": false,
- "type": "boolean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/Webhook.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/Webhook.json
deleted file mode 100644
index 4fe2705..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/Webhook.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.Webhook",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/webhooks/reference/get/account_activity/webhooks",
- "properties": {
- "id": {
- "type": "string"
- },
- "url": {
- "type": "string"
- },
- "valid": {
- "type": "boolean"
- },
- "created_timestamp": {
- "type": "string",
- "format": "date-time"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageNewRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageNewRequest.json
deleted file mode 100644
index 7eadf76..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageNewRequest.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.WelcomeMessageNewRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/post/direct_messages/welcome_messages/new",
- "properties": {
- "welcome_message": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.api.WelcomeMessageNewRequestWrapper",
- "properties": {
- "message_data": {
- "$ref": "../pojo/MessageData.json"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageNewResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageNewResponse.json
deleted file mode 100644
index 7062059..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageNewResponse.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.WelcomeMessageNewResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/post/direct_messages/welcome_messages/new",
- "properties": {
- "welcome_message": {
- "$ref": "../pojo/WelcomeMessage.json"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageNewRuleRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageNewRuleRequest.json
deleted file mode 100644
index c54ed7e..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageNewRuleRequest.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.WelcomeMessageNewRuleRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/post/direct_messages/welcome_messages/rules/new",
- "properties": {
- "welcome_message_rule": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.api.WelcomeMessageNewRuleRequestWrapper",
- "properties": {
- "welcome_message_id": {
- "description": "The ID of the Welcome Message that will be sent when this Rule is triggered.",
- "type": "string",
- "required": true
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageNewRuleResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageNewRuleResponse.json
deleted file mode 100644
index 82921f4..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageNewRuleResponse.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.WelcomeMessageNewRuleResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/post/direct_messages/welcome_messages/rules/new",
- "properties": {
- "welcome_message_rule": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.api.WelcomeMessageNewRuleResponseWrapper",
- "properties": {
- "id": {
- "description": "The ID of the Welcome Message that will be sent when this Rule is triggered.",
- "type": "string",
- "required": true
- },
- "created_timestamp": {
- "description": "created_timestamp",
- "type": "string",
- "required": true
- },
- "welcome_message_id": {
- "description": "The ID of the Welcome Message that will be sent when this Rule is triggered.",
- "type": "string",
- "required": true
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageRulesListRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageRulesListRequest.json
deleted file mode 100644
index 0a585c7..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageRulesListRequest.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.WelcomeMessageRulesListRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/get/direct_messages/welcome_messages/list",
- "properties": {
- "count": {
- "description": "Max number of events to be returned. 20 default. 50 max.",
- "type": "integer",
- "required": false
- },
- "cursor": {
- "description": "For paging through result sets greater than 1 page, use the “next_cursor” property from the previous request.",
- "type": "string",
- "required": false
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageRulesListResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageRulesListResponse.json
deleted file mode 100644
index ec89ccb..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessageRulesListResponse.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.WelcomeMessageRulesListResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/get/direct_messages/welcome_messages/list",
- "properties": {
- "welcome_message_rules": {
- "type": "array",
- "items": {
- "$ref": "../pojo/WelcomeMessageRule.json"
- }
- },
- "next_curser": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessagesListRequest.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessagesListRequest.json
deleted file mode 100644
index 1a501ee..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessagesListRequest.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.WelcomeMessagesListRequest",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/get/direct_messages/welcome_messages/list",
- "properties": {
- "count": {
- "description": "Max number of events to be returned. 20 default. 50 max.",
- "type": "integer",
- "required": false
- },
- "cursor": {
- "description": "For paging through result sets greater than 1 page, use the “next_cursor” property from the previous request.",
- "type": "string",
- "required": false
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessagesListResponse.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessagesListResponse.json
deleted file mode 100644
index f42b653..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/api/WelcomeMessagesListResponse.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.WelcomeMessagesListResponse",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/get/direct_messages/welcome_messages/list",
- "properties": {
- "welcome_messages": {
- "type": "array",
- "items": {
- "$ref": "../pojo/WelcomeMessage.json"
- }
- },
- "next_curser": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterConfiguration.json
deleted file mode 100644
index d69468f..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterConfiguration.json
+++ /dev/null
@@ -1,84 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "protocol": {
- "type": "string",
- "description": "The protocol"
- },
- "host": {
- "type": "string",
- "description": "The host"
- },
- "port": {
- "type": "integer",
- "description": "The port"
- },
- "version": {
- "type": "string",
- "description": "The version"
- },
- "endpoint": {
- "type": "string",
- "description": "The endpoint"
- },
- "jsonStoreEnabled": {
- "type": "boolean"
- },
- "oauth": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.twitter.TwitterOAuthConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "appName": {
- "type": "string"
- },
- "consumerKey": {
- "type": "string"
- },
- "consumerSecret": {
- "type": "string"
- },
- "accessToken": {
- "type": "string"
- },
- "accessTokenSecret": {
- "type": "string"
- }
- }
- },
- "basicauth": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.twitter.TwitterBasicAuthConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "username": {
- "type": "string"
- },
- "password": {
- "type": "string"
- }
- }
- },
- "threadsPerProvider": {
- "type": "integer",
- "description": "number of threads per provider"
- },
- "retrySleepMs": {
- "type": "integer",
- "description": "ms to sleep when hitting a rate limit"
- },
- "retryMax": {
- "type": "integer",
- "description": "max times to retry"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterFollowingConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterFollowingConfiguration.json
deleted file mode 100644
index 89fc7af..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterFollowingConfiguration.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterFollowingConfiguration",
- "extends": {"$ref":"TwitterUserInformationConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "ids_only": {
- "type": "boolean",
- "description": "Whether to collect ids only, or full profiles",
- "default": "true"
- },
- "max_items": {
- "type": "integer",
- "description": "Max items per user to collect",
- "default": 50000
- },
- "max_pages": {
- "type": "integer",
- "description": "Max pages per user to request",
- "default": 10
- },
- "page_size": {
- "type": "integer",
- "description": "Max items per page to request",
- "default": 5000
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterStreamConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterStreamConfiguration.json
deleted file mode 100644
index 6fa2a73..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterStreamConfiguration.json
+++ /dev/null
@@ -1,45 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterStreamConfiguration",
- "extends": {"$ref":"TwitterConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "includeEntities": {
- "type": "string"
- },
- "truncated": {
- "type": "boolean"
- },
- "filter-level": {
- "type": "string",
- "description": "Setting this parameter to one of none, low, or medium will set the minimum value of the filter_level Tweet attribute required to be included in the stream"
- },
- "with": {
- "type": "string",
- "description": "Typically following or user"
- },
- "replies": {
- "type": "string",
- "description": "Set to all, to see all @replies"
- },
- "follow": {
- "type": "array",
- "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream",
- "items": {
- "type": "integer"
- }
- },
- "track": {
- "type": "array",
- "description": "A list of phrases which will be used to determine what Tweets will be delivered on the stream",
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterTimelineProviderConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterTimelineProviderConfiguration.json
deleted file mode 100644
index 37ed60e..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterTimelineProviderConfiguration.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterTimelineProviderConfiguration",
- "extends": {"$ref":"TwitterUserInformationConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "max_items": {
- "type": "integer",
- "description": "Max items per user to collect",
- "default": 3200
- },
- "max_pages": {
- "type": "integer",
- "description": "Max items per page to request",
- "default": 16
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterUserInformationConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterUserInformationConfiguration.json
deleted file mode 100644
index 405c87a..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/config/TwitterUserInformationConfiguration.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.TwitterUserInformationConfiguration",
- "extends": {"$ref":"TwitterConfiguration.json"},
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "info": {
- "type": "array",
- "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream",
- "items": {
- "type": "string"
- }
- },
- "page_size": {
- "type": "integer",
- "description": "Max items per page to request",
- "default": 200
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/AccountSettings.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/AccountSettings.json
deleted file mode 100644
index 1f47385..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/AccountSettings.json
+++ /dev/null
@@ -1,111 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType": "org.apache.streams.twitter.api.AccountSettings",
- "javaInterfaces": [
- "java.io.Serializable"
- ],
- "description": "https://dev.twitter.com/rest/reference/get/account/settings",
- "properties": {
- "always_use_https": {
- "type": "boolean"
- },
- "discoverable_by_email": {
- "type": "boolean"
- },
- "geo_enabled": {
- "type": "boolean"
- },
- "language": {
- "type": "string"
- },
- "language": {
- "type": "string"
- },
- "protected": {
- "type": "boolean"
- },
- "screen_name": {
- "type": "string"
- },
- "show_all_inline_media": {
- "type": "boolean"
- },
- "sleep_time": {
- "type": "object",
- "properties": {
- "enabled": {
- "type": "boolean"
- },
- "end_time": {
- "type": "string",
- "format": "date-time"
- },
- "start_time": {
- "type": "string",
- "format": "date-time"
- }
- }
- },
- "time_zone": {
- "type": "object",
- "properties": {
- "name": {
- "type": "string"
- },
- "tzinfo_name": {
- "type": "string"
- },
- "utc_offset": {
- "type": "integer"
- }
- }
- },
- "trend_location": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "country": {
- "type": "string"
- },
- "countryCode": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "parentid": {
- "type": "integer"
- },
- "placeType": {
- "type": "object",
- "properties": {
- "code": {
- "type": "integer"
- },
- "name": {
- "type": "string"
- }
- }
- },
- "url": {
- "type": "string"
- },
- "woeid": {
- "type": "integer"
- }
- }
- }
- },
- "use_cookie_personalization": {
- "type": "boolean"
- },
- "allow_contributor_request": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/CallToAction.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/CallToAction.json
deleted file mode 100644
index 148eb62..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/CallToAction.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.CallToAction",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "type": {
- "type": "string",
- "default": "web_url",
- "required": true,
- "description": "Defines the type of button to display. Currently must be set to web_url."
- },
- "label": {
- "type": "string",
- "required": true,
- "description": "The text that will be displayed to the user on each button. Max string length of 36 characters."
- },
- "url": {
- "type": "string",
- "required": true,
- "description": "A valid http or https target URL of the button."
- },
- "tco_url": {
- "type": "string",
- "required": false,
- "description": "The t.co version of the URL will be returned in a POST response and on the read path (GET requests) only."
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json
deleted file mode 100644
index 03b801d..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.pojo.Delete",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "delete": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.DeleteDetails",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "status": {
- "type": "object",
- "properties": {
- "id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "user_id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "id_str": {
- "type": "string"
- },
- "user_id_str": {
- "type": "string"
- }
- }
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/DirectMessageEvent.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/DirectMessageEvent.json
deleted file mode 100644
index c6d39e1..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/DirectMessageEvent.json
+++ /dev/null
@@ -1,61 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.DirectMessageEvent",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "id": {
- "type": "string"
- },
- "created_timestamp": {
- "type": "string"
- },
- "type": {
- "type": "string",
- "default": "message_create",
- "enum" : [
- "message_create"
- ]
- },
- "initiated_via": {
- "type": "object",
- "properties": {
- "tweet_id": {
- "description": "The ID of the Tweet with Direct Message Prompt the event was initiated from if one was used.",
- "type": "string"
- },
- "welcome_message_id": {
- "description": "The ID of the Welcome Message immediately preceding the event if one was used.",
- "type": "string"
- }
- }
- },
- "message_create": {
- "type": "object",
- "properties": {
- "sender_id": {
- "type": "string"
- },
- "source_app_id": {
- "type": "string"
- },
- "target": {
- "type": "object",
- "properties": {
- "recipient_id": {
- "type": "string"
- }
- }
- },
- "message_data": {
- "$ref": "../pojo/MessageData.json"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json
deleted file mode 100644
index 320db12..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.pojo.Follow",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "follower": {
- "$ref": "User.json"
- },
- "followee": {
- "$ref": "User.json"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json
deleted file mode 100644
index ae7a021..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.FriendList",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "friends": {
- "type": "array",
- "items": {
- "type": "integer"
- }
- },
- "friends_str": {
- "type": "array",
- "items": {
- "type": "string"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Friendship.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Friendship.json
deleted file mode 100644
index cf42548..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Friendship.json
+++ /dev/null
@@ -1,39 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.Friendship",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "Detailed information about the relationship between the authenticated user and another user.",
- "properties": {
- "name": {
- "type": "string"
- },
- "screen_name": {
- "type": "string"
- },
- "id": {
- "type": "integer"
- },
- "id_str": {
- "type": "string"
- },
- "connections": {
- "type": "array",
- "items": {
- "type": "string",
- "enum": [
- "following",
- "following_requested",
- "followed_by",
- "none",
- "blocking",
- "muting"
- ]
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/MediaItem.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/MediaItem.json
deleted file mode 100644
index 4d11a6d..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/MediaItem.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.MediaItem",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "id": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/MessageAttachment.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/MessageAttachment.json
deleted file mode 100644
index b79229e..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/MessageAttachment.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.MediaItem",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "type": {
- "type": "string"
- },
- "media": {
- "$ref": "./MediaItem.json"
- }
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/MessageData.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/MessageData.json
deleted file mode 100644
index 0df1369..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/MessageData.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.MessageData",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "text": {
- "type": "string"
- },
- "ctas": {
- "type": "array",
- "items": {
- "$ref": "./CallToAction.json"
- }
- },
- "entities": {
- "$ref": "./tweet.json/#/properties/entities"
- },
- "quick_reply_response": {
- "$ref": "./QuickReply.json"
- },
- "attachment": {
- "$ref": "./MessageAttachment.json"
- }
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Place.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Place.json
deleted file mode 100644
index 63ac3fc..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Place.json
+++ /dev/null
@@ -1,99 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.Place",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "Places are specific, named locations with corresponding geo coordinates. They can be attached to Tweets by specifying a place_id when tweeting. Tweets associated with places are not necessarily issued from that location but could also potentially be about that location. Places can be searched for. Tweets can also be found by place_id. Places also have an attributes field that further describes a Place. These attributes are more convention rather than standard practic [...]
- "properties": {
- "id": {
- "type": "string"
- },
- "attributes": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.PlaceAttributes",
- "properties": {
- "street_address": {
- "type": "string"
- },
- "locality": {
- "type": "string",
- "description": "the city the place is in"
- },
- "region": {
- "type": "string",
- "description": "the administrative region the place is in"
- },
- "iso3": {
- "type": "string",
- "description": "the country code"
- },
- "postal_code": {
- "type": "string",
- "description": "the country code"
- },
- "phone": {
- "type": "string",
- "description": "in the preferred local format for the place, include long distance code"
- },
- "twitter": {
- "type": "string",
- "description": "twitter screen-name, without @"
- },
- "url": {
- "type": "string",
- "description": "official/canonical URL for place"
- }
- }
- },
- "bounding_box": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.BoundingBox",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "type": {
- "type": "string"
- },
- "coordinates": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "type": "number"
- }
- }
- }
- }
- }
- },
- "country": {
- "description": "Name of the country containing this place.",
- "type": "string"
- },
- "country_code": {
- "description": "Shortened country code representing the country containing this place.",
- "type": "string"
- },
- "full_name": {
- "description": "Full human-readable representation of the place’s name.",
- "type": "string"
- },
- "name": {
- "description": "Short human-readable representation of the place’s name.",
- "type": "string"
- },
- "place_type": {
- "description": "The type of location represented by this place.",
- "type": "string"
- },
- "url": {
- "description": "URL representing the location of additional place metadata for this place.",
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/QuickReply.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/QuickReply.json
deleted file mode 100644
index 1cbc8d6..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/QuickReply.json
+++ /dev/null
@@ -1,117 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.QuickReply",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "type": {
- "type": "string",
- "enum" : [
- "options",
- "location",
- "text_input"
- ]
- },
- "text_input": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.pojo.QuickReplyTextInput",
- "properties": {
- "label": {
- "type": "string",
- "description": "The hint text in the text input. String, max length of 36 characters including spaces.",
- "required": true
- },
- "keyboard": {
- "type": "string",
- "description": "The predefined keyboard type. String, value of “default” (standard text entry) or “number” (numeric entry).",
- "required": false
- },
- "metadata": {
- "type": "string",
- "description": "Metadata that will be sent back in the webhook request. String, max length of 1,000 characters including spaces.",
- "required": false
- }
- }
- },
- "options": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.pojo.QuickReplyOption",
- "properties": {
- "label": {
- "type": "string",
- "description": "The text label displayed on the button face. Label text is returned as the user’s message response. String, max length of 36 characters including spaces. Values with URLs are not allowed and will return an error.",
- "required": true
- },
- "description": {
- "type": "string",
- "description": "Optional description text displayed under label text. All options must have this property defined if property is present in any option. Text is auto-wrapped and will display on a max of two lines and supports \n for controling line breaks. Description text is not include in the user’s message response. String, max length of 72 characters including spaces.",
- "required": false
- },
- "metadata": {
- "type": "string",
- "description": "Metadata that will be sent back in the webhook request. String, max length of 1,000 characters including spaces.",
- "required": false
- }
- }
- }
- },
- "location": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.pojo.QuickReplyTextInput",
- "properties": {
- "type": {
- "type": "string",
- "description": "String value of shared_coordinate or shared_place",
- "required": true,
- "enum": [
- "shared_coordinate",
- "shared_place"
- ]
- },
- "shared_coordinate": {
- "type": "object",
- "properties": {
- "type": {
- "type": "string",
- "enum": [
- "Point"
- ]
- },
- "coordinates": {
- "type": "array",
- "items": {
- "type": "number"
- }
- }
- }
- },
- "shared_place": {
- "type": "object",
- "properties": {
- "place": {
- "type": "object",
- "properties": {
- "id": {
- "description": "Not all place IDs are discoverable with `GET geo/search </rest/reference/get/geo/search>`_. To retrieve all available details for a place ID you may also use `GET geo/id/:place_id </rest/reference/get/geo/id/place_id>`_.",
- "type": "string"
- }
- }
- }
- }
- },
- "metadata": {
- "type": "string",
- "description": "Metadata that will be sent back in the webhook request. String, max length of 1,000 characters including spaces.",
- "required": false
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Relationship.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Relationship.json
deleted file mode 100644
index 9e61381..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Relationship.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.Relationship",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "Detailed information about the relationship between two arbitrary users.",
- "properties": {
- "source": {
- "$ref": "./RelationshipSource.json"
- },
- "target": {
- "$ref": "./RelationshipTarget.json"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/RelationshipSource.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/RelationshipSource.json
deleted file mode 100644
index 2123a15..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/RelationshipSource.json
+++ /dev/null
@@ -1,49 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.RelationshipSource",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "can_dm": {
- "type": "boolean"
- },
- "blocking": {
- "type": "object"
- },
- "muting": {
- "type": "object"
- },
- "id_str": {
- "type": "string"
- },
- "all_replies": {
- "type": "object"
- },
- "want_retweets": {
- "type": "object"
- },
- "id": {
- "type": "integer"
- },
- "marked_spam": {
- "type": "object"
- },
- "screen_name": {
- "type": "string"
- },
- "following": {
- "type": "boolean"
- },
- "followed_by": {
- "type": "boolean"
- },
- "notifications_enabled": {
- "type": "object"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/RelationshipTarget.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/RelationshipTarget.json
deleted file mode 100644
index 64d05d9..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/RelationshipTarget.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.RelationshipTarget",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "type": "integer"
- },
- "id_str": {
- "type": "string"
- },
- "screen_name": {
- "type": "string"
- },
- "following": {
- "type": "boolean"
- },
- "followed_by": {
- "type": "boolean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json
deleted file mode 100644
index a449525..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.Retweet",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {"$ref":"tweet.json"},
- "properties": {
- "retweeted_status": {
- "type": "object",
- "required" : false,
- "description" : "Describes the tweet being retweeted.",
- "$ref" : "tweet.json"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json
deleted file mode 100644
index c6cb798..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json
+++ /dev/null
@@ -1,129 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.pojo.User",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "location": {
- "type": "string"
- },
- "default_profile": {
- "type": "boolean"
- },
- "statuses_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "profile_background_tile": {
- "type": "boolean"
- },
- "lang": {
- "type": "string"
- },
- "profile_link_color": {
- "type": "string"
- },
- "id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "protected": {
- "type": "boolean"
- },
- "favourites_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "profile_text_color": {
- "type": "string"
- },
- "verified": {
- "type": "boolean"
- },
- "description": {
- "type": "string"
- },
- "contributors_enabled": {
- "type": "boolean"
- },
- "name": {
- "type": "string"
- },
- "profile_sidebar_border_color": {
- "type": "string"
- },
- "profile_background_color": {
- "type": "string"
- },
- "created_at": {
- "type": "string",
- "format": "date-time"
- },
- "default_profile_image": {
- "type": "boolean"
- },
- "followers_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "geo_enabled": {
- "type": "boolean"
- },
- "profile_image_url_https": {
- "type": "string"
- },
- "profile_background_image_url": {
- "type": "string"
- },
- "profile_background_image_url_https": {
- "type": "string"
- },
- "follow_request_sent": {
- "type": "boolean"
- },
- "url": {
- "type": "string"
- },
- "utc_offset": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "time_zone": {
- "type": "string"
- },
- "profile_use_background_image": {
- "type": "boolean"
- },
- "friends_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "profile_sidebar_fill_color": {
- "type": "string"
- },
- "screen_name": {
- "type": "string"
- },
- "id_str": {
- "type": "string"
- },
- "profile_image_url": {
- "type": "string"
- },
- "is_translator": {
- "type": "boolean"
- },
- "listed_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "status": {
- "$ref": "tweet.json"
- }
- }
-
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json
deleted file mode 100644
index 23f8365..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json
+++ /dev/null
@@ -1,52 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.UserstreamEvent",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "created_at": {
- "type": "string",
- "format" : "date-time"
- },
- "event_type": {
- "type": "string",
- "enum" : [
- "access_revoked",
- "block",
- "unblock",
- "favorite",
- "unfavorite",
- "follow",
- "unfollow",
- "list_created",
- "list_destroyed",
- "list_updated",
- "list_member_added",
- "list_member_removed",
- "list_user_subscribed",
- "list_user_unsubscribed",
- "user_update"
- ]
- },
- "source": {
- "type": "string",
- "items": {
- "type": "integer"
- }
- },
- "target": {
- "type": "string",
- "items": {
- "type": "integer"
- }
- },
- "target_object": {
- "type": "object"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/WebhookEvents.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/WebhookEvents.json
deleted file mode 100644
index a72d5f6..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/WebhookEvents.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.WebhookEvents",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "direct_message_events": {
- "type": "array",
- "items": {
- "$ref": "./DirectMessageEvent.json"
- }
- },
- "users": {
- "type": "object"
- },
- "apps": {
- "type": "object"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/WelcomeMessage.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/WelcomeMessage.json
deleted file mode 100644
index 2867463..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/WelcomeMessage.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.WelcomeMessage",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "id": {
- "type": "string"
- },
- "created_timestamp": {
- "type": "string",
- "format" : "date-time"
- },
- "message_data": {
- "$ref": "../pojo/MessageData.json"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/WelcomeMessageRule.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/WelcomeMessageRule.json
deleted file mode 100644
index 07eee0c..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/WelcomeMessageRule.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.WelcomeMessageRule",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "",
- "properties": {
- "id": {
- "type": "string"
- },
- "created_timestamp": {
- "type": "string",
- "format" : "date-time"
- },
- "welcome_message_id": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json
deleted file mode 100644
index e529559..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json
+++ /dev/null
@@ -1,219 +0,0 @@
-{
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "javaType" : "org.apache.streams.twitter.pojo.Tweet",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "text": {
- "type": "string"
- },
- "retweeted": {
- "type": "boolean"
- },
- "in_reply_to_screen_name": {
- "type": "string"
- },
- "truncated": {
- "type": "boolean"
- },
- "filter_level": {
- "type": "string"
- },
- "contributors": {
- "ignore_malformed": false,
- "type": "array",
- "items": [{
- "type" : "number"
- }]
- },
- "place": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.Place",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "type": "string"
- },
- "bounding_box": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.BoundingBox",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "type": {
- "type": "string"
- },
- "coordinates": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "type": "array",
- "items": {
- "type": "number"
- }
- }
- }
- }
- }
- }
- }
- },
- "entities": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.twitter.pojo.Entities",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "user_mentions": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType" : "org.apache.streams.twitter.pojo.UserMentions",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "name": {
- "type": "string"
- },
- "indices": {
- "type": "array",
- "items": [{
- "type" : "integer"
- }]
- },
- "screen_name": {
- "type": "string"
- },
- "id_str": {
- "type": "string"
- }
- }
- }
- },
- "hashtags": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.pojo.Hashtag",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "text": {
- "type": "string"
- },
- "indices": {
- "type": "array",
- "items": [{
- "type": "integer"
- }]
- }
- }
- }
- },
- "urls": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "org.apache.streams.twitter.Url",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "expanded_url": {
- "type": "string"
- },
- "indices": {
- "type": "array",
- "items": [
- {
- "type" : "integer"
- }
- ]
- },
- "display_url": {
- "type": "string"
- },
- "url": {
- "type": "string"
- }
- }
- }
- }
- }
- },
- "in_reply_to_status_id_str": {
- "type": "string"
- },
- "id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "in_reply_to_user_id_str": {
- "type": "string"
- },
- "source": {
- "type": "string"
- },
- "lang": {
- "type": "string"
- },
- "favorited": {
- "type": "boolean"
- },
- "possibly_sensitive": {
- "type": "boolean"
- },
- "in_reply_to_status_id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "created_at": {
- "type" : "string",
- "format" : "date-time"
- },
- "in_reply_to_user_id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "favorite_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "retweet_count": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "id_str": {
- "type": "string"
- },
- "user": {
- "$ref": "User.json"
- },
- "is_quote_status": {
- "type": "boolean"
- },
- "quoted_status_id": {
- "ignore_malformed": false,
- "type": "integer"
- },
- "quoted_status": {
- "type": "object",
- "required" : false,
- "description" : "Describes the tweet being quoted.",
- "$ref" : "tweet.json"
- },
- "retweeted_status_id": {
- "type": "integer"
- },
- "retweeted_status": {
- "type": "object",
- "required" : false,
- "description" : "Describes the tweet being retweeted.",
- "$ref" : "tweet.json"
- }
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/resources/components.dot b/streams-contrib/streams-provider-twitter/src/main/resources/components.dot
deleted file mode 100644
index 98654b7..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/resources/components.dot
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- subgraph cluster_generators {
- label="generators";
- style="dotted";
- generators_twitter [label="https://api.twitter.com"]
- }
-
- subgraph cluster_providers {
- label="providers";
- providers_twitter_userstream [label="TwitterUserstreamProvider"]
- providers_twitter_userinfo [label="TwitterUserInformationProvider"]
- providers_twitter_timeline [label="TwitterTimelineProvider"]
- providers_twitter_following [label="TwitterFollowingProvider"]
-
- }
-
- subgraph cluster_processors {
- label="processors";
- processors_twitter_activity [label="ActivityConverterProcessor"]
- processors_twitter_activityobject [label="ActivityObjectConverterProcessor"]
- }
-
- persisters [label="persisters", shape="cylinder"];
-
- generators_twitter -> providers_twitter_userstream
- generators_twitter -> providers_twitter_timeline
- generators_twitter -> providers_twitter_following
- generators_twitter -> providers_twitter_userinfo
- providers_twitter_userinfo -> processors_twitter_activityobject [label="o.a.s.t.User"]
- providers_twitter_userstream -> processors_twitter_activity [label="o.a.s.t.Tweet"]
- providers_twitter_timeline -> processors_twitter_activity [label="o.a.s.t.Tweet"]
- providers_twitter_following -> processors_twitter_activity [label="o.a.s.t.Follow"]
-
- processors_twitter_activityobject -> persisters [label="o.a.s.p.j.Page"]
- processors_twitter_activity -> persisters [label="o.a.s.p.j.Post\no.a.s.p.j.Share\no.a.s.p.j.Follow"]
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/main/resources/followers.conf b/streams-contrib/streams-provider-twitter/src/main/resources/followers.conf
deleted file mode 100644
index 6ca5376..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/resources/followers.conf
+++ /dev/null
@@ -1,14 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-twitter {
- endpoint = followers
- oauth {
- consumerKey = ""
- consumerSecret = ""
- accessToken = ""
- accessTokenSecret = ""
- }
- info = [
- TheASF
- ]
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/resources/friends.conf b/streams-contrib/streams-provider-twitter/src/main/resources/friends.conf
deleted file mode 100644
index cc638d7..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/resources/friends.conf
+++ /dev/null
@@ -1,14 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-twitter {
- endpoint = friends
- oauth {
- consumerKey = ""
- consumerSecret = ""
- accessToken = ""
- accessTokenSecret = ""
- }
- info = [
- TheASF
- ]
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/resources/reference.conf b/streams-contrib/streams-provider-twitter/src/main/resources/reference.conf
deleted file mode 100644
index 0a9a2bd..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/resources/reference.conf
+++ /dev/null
@@ -1,18 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-twitter {
- protocol = "https"
- host = "api.twitter.com"
- port = 443
- version = "1.1"
- jsonStoreEnabled = true
- filter-level = "none"
- oauth {
- appName = "Apache Streams"
- }
- with = "user"
- replies = "all"
- threadsPerProvider = 10
- retrySleepMs = 1000
- retryMax = 10
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/resources/sample.conf b/streams-contrib/streams-provider-twitter/src/main/resources/sample.conf
deleted file mode 100644
index a6e6588..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/resources/sample.conf
+++ /dev/null
@@ -1,17 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-twitter {
- endpoint = sample
- oauth {
- consumerKey = ""
- consumerSecret = ""
- accessToken = ""
- accessTokenSecret = ""
- }
- track = [
- apache
- ]
- follow = [
- TheASF
- ]
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/resources/userinfo.conf b/streams-contrib/streams-provider-twitter/src/main/resources/userinfo.conf
deleted file mode 100644
index 74197c1..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/resources/userinfo.conf
+++ /dev/null
@@ -1,14 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-twitter {
- endpoint = userstream
- oauth {
- consumerKey = ""
- consumerSecret = ""
- accessToken = ""
- accessTokenSecret = ""
- }
- info = [
- TheASF
- ]
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/main/resources/userstream.conf b/streams-contrib/streams-provider-twitter/src/main/resources/userstream.conf
deleted file mode 100644
index 1fb19ef..0000000
--- a/streams-contrib/streams-provider-twitter/src/main/resources/userstream.conf
+++ /dev/null
@@ -1,11 +0,0 @@
-# Apache Streams (incubating)
-# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
-twitter {
- endpoint = userstream
- oauth {
- consumerKey = ""
- consumerSecret = ""
- accessToken = ""
- accessTokenSecret = ""
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/site/markdown/index.md b/streams-contrib/streams-provider-twitter/src/site/markdown/index.md
deleted file mode 100644
index 4249956..0000000
--- a/streams-contrib/streams-provider-twitter/src/site/markdown/index.md
+++ /dev/null
@@ -1,54 +0,0 @@
-org.apache.streams:streams-provider-twitter
-===========================================
-
-streams-provider-twitter contains schema definitions, providers, conversions, and utility classes.
-
-## Data Types
-
-| Schema |
-|--------|
-| [Tweet.json](com/twitter/tweet.json "Tweet.json") [Tweet.html](apidocs/org/apache/streams/twitter/pojo/Tweet.html "javadoc") |
-| [Retweet.json](com/twitter/Retweet.json "Retweet.json") [Retweet.html](apidocs/org/apache/streams/twitter/pojo/Retweet.html "javadoc") |
-| [User.json](com/twitter/User.json "User.json") [User.html](apidocs/org/apache/streams/twitter/pojo/User.html "javadoc") |
-| [Delete.json](com/twitter/Delete.json "Delete.json") [Delete.html](apidocs/org/apache/streams/twitter/pojo/Delete.html "javadoc") |
-| [UserstreamEvent.json](com/twitter/UserstreamEvent.json "UserstreamEvent.json") [UserstreamEvent.html](apidocs/org/apache/streams/twitter/pojo/UserstreamEvent.html "javadoc") |
-| [FriendList.json](com/twitter/FriendList.json "FriendList.json") [FriendList.html](apidocs/org/apache/streams/twitter/pojo/FriendList.html "javadoc") |
-
-## Configuration
-
-| Schema |
-|--------|
-| [TwitterConfiguration.json](com/twitter/tweet.json "TwitterConfiguration.json") [TwitterConfiguration.html](apidocs/org/apache/streams/twitter/TwitterConfiguration.html "javadoc") |
-
-## Components
-
-
-
-| Class | Configuration | Example Configuration(s) |
-|-------|---------------|--------------------------|
-| TwitterUserInformationProvider [TwitterUserInformationProvider.html](apidocs/org/apache/streams/twitter/TwitterUserInformationConfiguration.html "javadoc") | [TwitterUserInformationConfiguration.json](com/twitter/TwitterUserInformationConfiguration.json "TwitterUserInformationConfiguration.json") [TwitterUserInformationConfiguration.html](apidocs/org/apache/streams/twitter/pojo/TwitterUserInformationConfiguration.html "javadoc") | [userinfo.conf](userinfo.conf "userinfo.conf") |
-| TwitterTimelineProvider [TwitterTimelineProvider.html](apidocs/org/apache/streams/twitter/TwitterTimelineConfiguration.html "javadoc") | [TwitterUserInformationConfiguration.json](com/twitter/TwitterUserInformationConfiguration.json "TwitterUserInformationConfiguration.json") [TwitterUserInformationConfiguration.html](apidocs/org/apache/streams/twitter/pojo/TwitterUserInformationConfiguration.html "javadoc") | [userinfo.conf](userinfo.conf "userinfo.conf") |
-| TwitterStreamProvider [TwitterStreamProvider.html](apidocs/org/apache/streams/twitter/TwitterStreamProvider.html "javadoc") | [TwitterStreamConfiguration.json](com/twitter/TwitterStreamConfiguration.json "TwitterStreamConfiguration.json") [TwitterUserInformationConfiguration.html](apidocs/org/apache/streams/twitter/pojo/TwitterStreamConfiguration.html "javadoc") | [sample.conf](sample.conf "sample.conf")<br/>[userstream.conf](userstream.conf "userstream.conf") |
-| TwitterFollowingProvider [TwitterFollowingProvider.html](apidocs/org/apache/streams/twitter/TwitterFollowingConfiguration.html "javadoc") | [TwitterFollowingConfiguration.json](com/twitter/TwitterFollowingConfiguration.json "TwitterFollowingConfiguration.json") [TwitterFollowingConfiguration.html](apidocs/org/apache/streams/twitter/pojo/TwitterFollowingConfiguration.html "javadoc") | [friends.conf](friends.conf "friends.conf")<br/>[followers.conf](followers.conf "followers.conf") |
-
-Test:
------
-
-Create a local file `application.conf` with valid twitter credentials
-
- twitter {
- oauth {
- consumerKey = ""
- consumerSecret = ""
- accessToken = ""
- accessTokenSecret = ""
- }
- }
-
-Build with integration testing enabled, using your credentials
-
- mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/application.conf"
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-provider-twitter/src/site/markdown/twitter.md b/streams-contrib/streams-provider-twitter/src/site/markdown/twitter.md
deleted file mode 100644
index 4960ccb..0000000
--- a/streams-contrib/streams-provider-twitter/src/site/markdown/twitter.md
+++ /dev/null
@@ -1,24 +0,0 @@
-## Twitter Credentials
-
-Create a local file `twitter.oauth.conf` with valid twitter credentials
-
- twitter {
- oauth {
- consumerKey = ""
- consumerSecret = ""
- accessToken = ""
- accessTokenSecret = ""
- }
- }
-
-Log into developer.twitter.com
-
-Visit https://apps.twitter.com and create an application.
-
-Select your application and click into 'Keys and Access Tokens'
-
-The consumerKey and consumerSecret are application-wide.
-
-The accessToken and accessTokenSecret are per-user.
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-provider-twitter/src/site/site.xml b/streams-contrib/streams-provider-twitter/src/site/site.xml
deleted file mode 100644
index 1296d0e..0000000
--- a/streams-contrib/streams-provider-twitter/src/site/site.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project>
- <body>
- <links name="Help">
- <item name="Twitter" href="twitter.html"/>
- </links>
- </body>
-</project>
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/api/TwitterIT.java b/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/api/TwitterIT.java
deleted file mode 100644
index c297b8d..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/api/TwitterIT.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.test.api;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.twitter.TwitterConfiguration;
-import org.apache.streams.twitter.api.Account;
-import org.apache.streams.twitter.api.AccountActivity;
-import org.apache.streams.twitter.api.AccountSettings;
-import org.apache.streams.twitter.api.DirectMessages;
-import org.apache.streams.twitter.api.EventsListRequest;
-import org.apache.streams.twitter.api.EventsListResponse;
-import org.apache.streams.twitter.api.Favorites;
-import org.apache.streams.twitter.api.FavoritesListRequest;
-import org.apache.streams.twitter.api.Followers;
-import org.apache.streams.twitter.api.FollowersIdsRequest;
-import org.apache.streams.twitter.api.FollowersIdsResponse;
-import org.apache.streams.twitter.api.FollowersListRequest;
-import org.apache.streams.twitter.api.FollowersListResponse;
-import org.apache.streams.twitter.api.Friends;
-import org.apache.streams.twitter.api.FriendsIdsRequest;
-import org.apache.streams.twitter.api.FriendsIdsResponse;
-import org.apache.streams.twitter.api.FriendsListRequest;
-import org.apache.streams.twitter.api.FriendsListResponse;
-import org.apache.streams.twitter.api.Statuses;
-import org.apache.streams.twitter.api.StatusesHomeTimelineRequest;
-import org.apache.streams.twitter.api.StatusesLookupRequest;
-import org.apache.streams.twitter.api.StatusesMentionsTimelineRequest;
-import org.apache.streams.twitter.api.StatusesShowRequest;
-import org.apache.streams.twitter.api.Twitter;
-import org.apache.streams.twitter.api.Users;
-import org.apache.streams.twitter.api.UsersLookupRequest;
-import org.apache.streams.twitter.api.UsersShowRequest;
-import org.apache.streams.twitter.api.Webhook;
-import org.apache.streams.twitter.api.WelcomeMessageRules;
-import org.apache.streams.twitter.api.WelcomeMessageRulesListRequest;
-import org.apache.streams.twitter.api.WelcomeMessageRulesListResponse;
-import org.apache.streams.twitter.api.WelcomeMessages;
-import org.apache.streams.twitter.api.WelcomeMessagesListRequest;
-import org.apache.streams.twitter.api.WelcomeMessagesListResponse;
-import org.apache.streams.twitter.pojo.Tweet;
-import org.apache.streams.twitter.pojo.User;
-import org.apache.streams.twitter.pojo.WelcomeMessage;
-import org.apache.streams.twitter.pojo.WelcomeMessageRule;
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import org.apache.juneau.remoteable.RemoteMethod;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static java.util.Objects.nonNull;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-/**
- * Integration Tests for all implemented Twitter endpoints.
- */
-public class TwitterIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterIT.class);
-
- private static Config application = ConfigFactory.parseResources("TwitterIT.conf").withFallback(ConfigFactory.load());
- private static StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(application);
- private static TwitterConfiguration config = new ComponentConfigurator<>(TwitterConfiguration.class).detectConfiguration(application, "twitter");
-
- private static User user;
- private static AccountSettings settings;
- private static List<Tweet> statusesHomeTimeline;
-
- @Test(groups = {"Account","AccountVerifyCredentials"})
- public void testVerifyCredentials() throws Exception {
- Account account = Twitter.getInstance(config);
- nonNull(account);
- User user = account.verifyCredentials();
- nonNull(user);
- nonNull(user.getCreatedAt());
- TwitterIT.user = user;
- }
-
- @Test(dependsOnGroups = {"AccountVerifyCredentials"}, groups = {"Account"})
- public void testAccountSettings() throws Exception {
- Account account = Twitter.getInstance(config);
- nonNull(account);
- AccountSettings settings = account.settings();
- nonNull(settings);
- nonNull(settings.getScreenName());
- assertEquals(settings.getScreenName(), user.getScreenName());
- TwitterIT.settings = settings;
- }
-
- @Test(dependsOnGroups = {"Account"}, groups = {"Favorites"})
- public void testFavoritesList() throws Exception {
- Favorites favorites = Twitter.getInstance(config);
- nonNull(favorites);
- List<Tweet> favoritesTweetList = favorites.list(new FavoritesListRequest().withScreenName(user.getScreenName()));
- nonNull(favoritesTweetList);
- assertThat("favoritesTweetList.size() > 0", favoritesTweetList.size() > 0);
- }
-
- @Test(dependsOnGroups = {"Account"}, groups = {"Followers"})
- public void testFollowersList() throws Exception {
- Followers followers = Twitter.getInstance(config);
- nonNull(followers);
- FollowersListRequest followersListRequest = new FollowersListRequest();
- followersListRequest.setId(user.getId());
- FollowersListResponse followersListResponse = followers.list(followersListRequest);
- nonNull(followersListResponse);
- assertThat("followersListResponse.getUsers().size() > 0", followersListResponse.getUsers().size() > 0);
- }
-
- @Test(dependsOnGroups = {"Account"}, groups = {"Followers"})
- public void testFollowersIds() throws Exception {
- Followers followers = Twitter.getInstance(config);
- nonNull(followers);
- FollowersIdsRequest followersIdsRequest = new FollowersIdsRequest();
- followersIdsRequest.setId(user.getId());
- FollowersIdsResponse followersIdsResponse = followers.ids(followersIdsRequest);
- nonNull(followersIdsResponse);
- assertThat("followersIdsResponse.getUsers().size() > 0", followersIdsResponse.getIds().size() > 0);
- }
-
- @Test(dependsOnGroups = {"Account"}, groups = {"Friends"})
- public void testFriendsList() throws Exception {
- Friends friends = Twitter.getInstance(config);
- nonNull(friends);
- FriendsListRequest friendsListRequest = (FriendsListRequest) (new FriendsListRequest().withId(user.getId()));
- FriendsListResponse friendsListResponse = friends.list(friendsListRequest);
- nonNull(friendsListResponse);
- assertThat("friendsListResponse.getUsers().size() > 0", friendsListResponse.getUsers().size() > 0);
- }
-
- @Test(dependsOnGroups = {"Account"}, groups = {"Friends"})
- public void testFriendsIds() throws Exception {
- Friends friends = Twitter.getInstance(config);
- nonNull(friends);
- FriendsIdsRequest friendsIdsRequest = (FriendsIdsRequest)new FriendsIdsRequest().withId(user.getId());
- FriendsIdsResponse friendsIdsResponse = friends.ids(friendsIdsRequest);
- nonNull(friendsIdsResponse);
- assertThat("friendsIdsResponse.getUsers().size() > 0", friendsIdsResponse.getIds().size() > 0);
- }
-
- @Test(dependsOnGroups = {"Account"}, groups = {"Statuses","StatusesHomeTimeline"})
- public void testStatusesHomeTimeline() throws Exception {
- Statuses statuses = Twitter.getInstance(config);
- nonNull(statuses);
- StatusesHomeTimelineRequest statusesHomeTimelineRequest = new StatusesHomeTimelineRequest();
- List<Tweet> statusesHomeTimeline = statuses.homeTimeline(statusesHomeTimelineRequest);
- nonNull(statusesHomeTimeline);
- assertThat("statusesHomeTimeline.size() > 0", statusesHomeTimeline.size() > 0);
- this.statusesHomeTimeline = statusesHomeTimeline;
- }
-
- @Test(dependsOnGroups = {"Account"}, groups = {"Statuses"})
- public void testStatusesMentionsTimeline() throws Exception {
- Statuses statuses = Twitter.getInstance(config);
- nonNull(statuses);
- StatusesMentionsTimelineRequest statusesMentionsTimelineRequest = new StatusesMentionsTimelineRequest();
- List<Tweet> statusesMentionsTimeline = statuses.mentionsTimeline(statusesMentionsTimelineRequest);
- nonNull(statusesMentionsTimeline);
- assertThat("statusesMentionsTimeline.size() > 0", statusesMentionsTimeline.size() > 0);
- }
-
- @Test(dependsOnGroups = {"Account","StatusesHomeTimeline"}, groups = {"Statuses"})
- public void testStatusesLookup() throws Exception {
- Statuses statuses = Twitter.getInstance(config);
- nonNull(statuses);
- StatusesLookupRequest statusesLookupRequest = new StatusesLookupRequest();
- statusesLookupRequest.setId(statusesHomeTimeline.stream().map(tweet -> tweet.getId()).collect(Collectors.toList()));
- List<Tweet> statusesLookup = statuses.lookup(statusesLookupRequest);
- nonNull(statusesLookup);
- assertThat("statusesLookup.size() > 0", statusesLookup.size() > 0);
- }
-
- @Test(dependsOnGroups = {"Account"}, groups = {"Statuses"})
- public void testStatusesShow() throws Exception {
- Statuses statuses = Twitter.getInstance(config);
- nonNull(statuses);
- StatusesShowRequest statusesShowRequest = new StatusesShowRequest();
- statusesShowRequest.setId(user.getStatus().getId());
- Tweet statusesShow = statuses.show(statusesShowRequest);
- nonNull(statusesShow);
- nonNull(statusesShow.getCreatedAt());
- }
-
- @Test(dependsOnGroups = {"Account"}, groups = {"Users"})
- public void testUsersShow() throws Exception {
- Users users = Twitter.getInstance(config);
- nonNull(users);
- User showUser = users.show(new UsersShowRequest().withScreenName(user.getScreenName()));
- nonNull(showUser);
- assertEquals( user.getId(), showUser.getId());
- }
-
- @Test(dependsOnGroups = {"Account"}, groups = {"Users"})
- public void testUsersLookupById() throws Exception {
- Users users = Twitter.getInstance(config);
- nonNull(users);
- UsersLookupRequest usersLookupRequest = new UsersLookupRequest();
- usersLookupRequest.setUserId(statusesHomeTimeline.stream().map(tweet -> tweet.getUser().getId()).collect(Collectors.toList()));
- List<User> lookupUserById = users.lookup(usersLookupRequest);
- nonNull(lookupUserById);
- assertThat("lookupUserById.size() > 0", lookupUserById.size() > 0);
- }
-
- @Test(dependsOnGroups = {"Account"}, groups = {"Users"})
- public void testUsersLookupByScreenName() throws Exception {
- Users users = Twitter.getInstance(config);
- nonNull(users);
- UsersLookupRequest usersLookupRequest = new UsersLookupRequest();
- usersLookupRequest.setScreenName(statusesHomeTimeline.stream().map(tweet -> tweet.getUser().getScreenName()).collect(Collectors.toList()));
- List<User> lookupUserByScreenName = users.lookup(usersLookupRequest);
- nonNull(lookupUserByScreenName);
- assertThat("lookupUserByScreenName.size() > 0", lookupUserByScreenName.size() > 0);
- }
-
- @Test(
- enabled=false,
- dependsOnGroups = {"Account"},
- groups = {"DirectMessages"}
- )
- public void testDirectMessagesListEvents() throws Exception {
- DirectMessages directMessages = Twitter.getInstance(config);
- nonNull(directMessages);
- EventsListResponse eventsListResponse = directMessages.listEvents(new EventsListRequest());
- nonNull(eventsListResponse);
- List<Object> events = eventsListResponse.getEvents();
- nonNull(events);
- assertThat("events.size() > 0", events.size() > 0);
- }
-
- @Test(
- enabled=false,
- dependsOnGroups = {"Account"},
- groups = {"AccountActivity"}
- )
- public void testGetWebhooks() throws Exception {
- AccountActivity accountActivity = Twitter.getInstance(config);
- nonNull(accountActivity);
- List<Webhook> webhooks = accountActivity.getWebhooks();
- nonNull(webhooks);
- }
-
- @Test(
- enabled=false,
- dependsOnGroups = {"Account"},
- groups = {"WelcomeMessages"}
- )
- public void testGetWelcomeMessages() throws Exception {
- WelcomeMessages welcomeMessages = Twitter.getInstance(config);
- nonNull(welcomeMessages);
- WelcomeMessagesListResponse welcomeMessageListResponse = welcomeMessages.listWelcomeMessages(new WelcomeMessagesListRequest());
- List<WelcomeMessage> welcomeMessageList = welcomeMessageListResponse.getWelcomeMessages();
- nonNull(welcomeMessageList);
- }
-
- @Test(
- enabled=false,
- dependsOnGroups = {"Account"},
- groups = {"WelcomeMessageRules"}
- )
- public void testGetWelcomeMessageRules() throws Exception {
- WelcomeMessageRules welcomeMessageRules = Twitter.getInstance(config);
- nonNull(welcomeMessageRules);
- WelcomeMessageRulesListResponse welcomeMessageListResponse = welcomeMessageRules.listWelcomeMessageRules(new WelcomeMessageRulesListRequest());
- List<WelcomeMessageRule> welcomeMessageRuleList = welcomeMessageListResponse.getWelcomeMessageRules();
- nonNull(welcomeMessageRuleList);
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/api/TwitterSecurityTest.java b/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/api/TwitterSecurityTest.java
deleted file mode 100644
index 873b113..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/api/TwitterSecurityTest.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.test.api;
-
-import org.apache.streams.twitter.TwitterOAuthConfiguration;
-import org.apache.streams.twitter.api.TwitterOAuthRequestInterceptor;
-import org.apache.streams.twitter.api.TwitterSecurity;
-import org.apache.streams.twitter.test.utils.TwitterActivityConvertersTest;
-
-import org.apache.http.HttpHost;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpRequestWrapper;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.protocol.HttpCoreContext;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URI;
-import java.security.MessageDigest;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.xml.bind.DatatypeConverter;
-
-import static org.junit.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-/**
- * Unit Tests for twitter request signing.
- */
-public class TwitterSecurityTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterActivityConvertersTest.class);
-
- private TwitterSecurity security = new TwitterSecurity();
-
- @Test
- public void testEncode() {
- assertEquals( "Ladies%20%2B%20Gentlemen", security.encode("Ladies + Gentlemen"));
- assertEquals( "An%20encoded%20string%21", security.encode("An encoded string!"));
- assertEquals( "Dogs%2C%20Cats%20%26%20Mice", security.encode("Dogs, Cats & Mice"));
- assertEquals( "%E2%98%83", security.encode("☃"));
- }
-
- @Test
- public void testGenerateSignatureBaseString() {
- Map<String,String> testParamMap = new HashMap<>();
- testParamMap.put("status", security.encode("Hello Ladies + Gentlemen, a signed OAuth request!"));
- testParamMap.put("include_entities", "true");
- testParamMap.put("oauth_consumer_key", "xvz1evFS4wEEPTGEFPHBog");
- testParamMap.put("oauth_nonce", "kYjzVBB8Y0ZFabxSWbWovY3uYSQ2pTgmZeNu2VS4cg");
- testParamMap.put("oauth_signature_method", "HMAC-SHA1");
- testParamMap.put("oauth_timestamp", "1318622958");
- testParamMap.put("oauth_token", "370773112-GmHxMAgYyLbNEtIKZeRNFsMKPR9EyMZeS9weJAEb");
- testParamMap.put("oauth_version", "1.0");
- String signature_parameter_string = security.generateSignatureParameterString(testParamMap);
- String signature_base_string = security.generateSignatureBaseString("POST", "https://api.twitter.com/1/statuses/update.json", signature_parameter_string);
- String expected = "POST&https%3A%2F%2Fapi.twitter.com%2F1%2Fstatuses%2Fupdate.json&include_entities%3Dtrue%26oauth_consumer_key%3Dxvz1evFS4wEEPTGEFPHBog%26oauth_nonce%3DkYjzVBB8Y0ZFabxSWbWovY3uYSQ2pTgmZeNu2VS4cg%26oauth_signature_method%3DHMAC-SHA1%26oauth_timestamp%3D1318622958%26oauth_token%3D370773112-GmHxMAgYyLbNEtIKZeRNFsMKPR9EyMZeS9weJAEb%26oauth_version%3D1.0%26status%3DHello%2520Ladies%2520%252B%2520Gentlemen%252C%2520a%2520signed%2520OAuth%2520request%2521";
- assertEquals(expected, signature_base_string);
- }
-
- /**
- * @see <a href=https://dev.twitter.com/oauth/overview/creating-signatures">https://dev.twitter.com/oauth/overview/creating-signatures</a>
- */
- @Test
- public void testGenerateSignatureParameterString() {
- Map<String,String> testParamMap = new HashMap<>();
- testParamMap.put("status", security.encode("Hello Ladies + Gentlemen, a signed OAuth request!"));
- testParamMap.put("include_entities", "true");
- testParamMap.put("oauth_consumer_key", "xvz1evFS4wEEPTGEFPHBog");
- testParamMap.put("oauth_nonce", "kYjzVBB8Y0ZFabxSWbWovY3uYSQ2pTgmZeNu2VS4cg");
- testParamMap.put("oauth_signature_method", "HMAC-SHA1");
- testParamMap.put("oauth_timestamp", "1318622958");
- testParamMap.put("oauth_token", "370773112-GmHxMAgYyLbNEtIKZeRNFsMKPR9EyMZeS9weJAEb");
- testParamMap.put("oauth_version", "1.0");
- String signature_parameter_string = security.generateSignatureParameterString(testParamMap);
- String expected = "include_entities=true&oauth_consumer_key=xvz1evFS4wEEPTGEFPHBog&oauth_nonce=kYjzVBB8Y0ZFabxSWbWovY3uYSQ2pTgmZeNu2VS4cg&oauth_signature_method=HMAC-SHA1&oauth_timestamp=1318622958&oauth_token=370773112-GmHxMAgYyLbNEtIKZeRNFsMKPR9EyMZeS9weJAEb&oauth_version=1.0&status=Hello%20Ladies%20%2B%20Gentlemen%2C%20a%20signed%20OAuth%20request%21";
- assertEquals(expected, signature_parameter_string);
- }
-
- @Test
- public void testGenerateAuthorizationHeaderString() {
- Map<String,String> oauthParamMap = new HashMap<>();
- oauthParamMap.put("oauth_consumer_key", "xvz1evFS4wEEPTGEFPHBog");
- oauthParamMap.put("oauth_nonce", "kYjzVBB8Y0ZFabxSWbWovY3uYSQ2pTgmZeNu2VS4cg");
- oauthParamMap.put("oauth_signature", "tnnArxj06cWHq44gCs1OSKk/jLY=");
- oauthParamMap.put("oauth_signature_method", "HMAC-SHA1");
- oauthParamMap.put("oauth_timestamp", "1318622958");
- oauthParamMap.put("oauth_token", "370773112-GmHxMAgYyLbNEtIKZeRNFsMKPR9EyMZeS9weJAEb");
- oauthParamMap.put("oauth_version", "1.0");
- String expected = "OAuth oauth_consumer_key=\"xvz1evFS4wEEPTGEFPHBog\", oauth_nonce=\"kYjzVBB8Y0ZFabxSWbWovY3uYSQ2pTgmZeNu2VS4cg\", oauth_signature=\"tnnArxj06cWHq44gCs1OSKk%2FjLY%3D\", oauth_signature_method=\"HMAC-SHA1\", oauth_timestamp=\"1318622958\", oauth_token=\"370773112-GmHxMAgYyLbNEtIKZeRNFsMKPR9EyMZeS9weJAEb\", oauth_version=\"1.0\"";
- String authorization_header_string = security.generateAuthorizationHeaderString(oauthParamMap);
- assertEquals(expected, authorization_header_string);
- }
-
- @Test
- public void testComputeOauthSignature() throws Exception {
- String signature_base_string = "POST&https%3A%2F%2Fapi.twitter.com%2F1%2Fstatuses%2Fupdate.json&include_entities%3Dtrue%26oauth_consumer_key%3Dxvz1evFS4wEEPTGEFPHBog%26oauth_nonce%3DkYjzVBB8Y0ZFabxSWbWovY3uYSQ2pTgmZeNu2VS4cg%26oauth_signature_method%3DHMAC-SHA1%26oauth_timestamp%3D1318622958%26oauth_token%3D370773112-GmHxMAgYyLbNEtIKZeRNFsMKPR9EyMZeS9weJAEb%26oauth_version%3D1.0%26status%3DHello%2520Ladies%2520%252B%2520Gentlemen%252C%2520a%2520signed%2520OAuth%2520request%2521";
- String signing_key = "kAcSOqF21Fu85e7zjz7ZN2U4ZRhfV3WpwPAoE3Z7kBw&LswwdoUaIvS8ltyTt5jkRh4J50vUPVVHtR2YPi5kE";
- String expected = "tnnArxj06cWHq44gCs1OSKk/jLY=";
- String oauth_signature = security.computeAndEncodeSignature(signature_base_string, signing_key, TwitterSecurity.oauth_signature_method);
- assertEquals(expected, oauth_signature);
- }
-
- @Test
- public void testComputeWebhookSignature() throws Exception {
- // https://tools.ietf.org/html/rfc4231
- String signature_base_string = "decacec0-51f5-11e7-98f5-7556589ff0ff";
- String signing_key = "uQ0gukfFwOZTQmTDRrZ35rwEe7D9WLXNYuZkCX147v8fYikK7u";
- String expected = "gPUsBnJwEylNWuiIpmwKeZKmh3tjm3sq1CXeGe/GD4M=";
- String webhook_signature = security.computeAndEncodeSignature(signature_base_string, signing_key, TwitterSecurity.webhook_signature_method);
- assertEquals(expected, webhook_signature);
- }
-
- @Test
- public void testProcess() throws Exception {
- URI testURI = new URIBuilder()
- .setPath("/1/statuses/update.json")
- .setParameter("include_entities", "true")
- .build();
- HttpPost testRequest = new HttpPost(testURI);
- testRequest.setEntity(new StringEntity("status="+security.encode("Hello Ladies + Gentlemen, a signed OAuth request!")));
- HttpHost host = new HttpHost("api.twitter.com", -1, "https");
- HttpRequestWrapper wrapper = HttpRequestWrapper.wrap(testRequest, host);
- TwitterOAuthConfiguration testOauthConfiguration = new TwitterOAuthConfiguration()
- .withConsumerKey("xvz1evFS4wEEPTGEFPHBog")
- .withConsumerSecret("kAcSOqF21Fu85e7zjz7ZN2U4ZRhfV3WpwPAoE3Z7kBw")
- .withAccessToken("370773112-GmHxMAgYyLbNEtIKZeRNFsMKPR9EyMZeS9weJAEb")
- .withAccessTokenSecret("LswwdoUaIvS8ltyTt5jkRh4J50vUPVVHtR2YPi5kE");
- TwitterOAuthRequestInterceptor interceptor = Mockito.spy(new TwitterOAuthRequestInterceptor(testOauthConfiguration));
- Mockito.when(interceptor.generateNonce()).thenReturn("kYjzVBB8Y0ZFabxSWbWovY3uYSQ2pTgmZeNu2VS4cg");
- Mockito.when(interceptor.generateTimestamp()).thenReturn("1318622958");
- interceptor.process(wrapper, new HttpCoreContext());
- assertEquals(1, wrapper.getHeaders("Authorization").length);
- String actual = wrapper.getFirstHeader("Authorization").getValue();
- String expected = "OAuth oauth_consumer_key=\"xvz1evFS4wEEPTGEFPHBog\", oauth_nonce=\"kYjzVBB8Y0ZFabxSWbWovY3uYSQ2pTgmZeNu2VS4cg\", oauth_signature=\"tnnArxj06cWHq44gCs1OSKk%2FjLY%3D\", oauth_signature_method=\"HMAC-SHA1\", oauth_timestamp=\"1318622958\", oauth_token=\"370773112-GmHxMAgYyLbNEtIKZeRNFsMKPR9EyMZeS9weJAEb\", oauth_version=\"1.0\"";
- assertEquals(expected, actual);
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/data/TwitterObjectMapperIT.java b/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/data/TwitterObjectMapperIT.java
deleted file mode 100644
index 481e965..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/data/TwitterObjectMapperIT.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.test.data;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.twitter.converter.TwitterDocumentClassifier;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.Tweet;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.apache.streams.twitter.converter.TwitterDateTimeFormat.TWITTER_FORMAT;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.greaterThan;
-
-/**
- * Tests serialization / deserialization of twitter jsons.
- */
-@Test(dependsOnGroups = {"Providers"}, groups = {"Data"})
-public class TwitterObjectMapperIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterObjectMapperIT.class);
-
- private static Config application = ConfigFactory.parseResources("TwitterObjectMapperIT.conf").withFallback(ConfigFactory.load());
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance(Stream.of(TWITTER_FORMAT).collect(Collectors.toList()));
-
- @Test(dependsOnGroups = "TwitterStreamProviderIT")
- public void tests() {
-
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
-
- String inputResourcePath = application.getString("inputResourcePath");
-
- InputStream is = TwitterObjectMapperIT.class.getResourceAsStream(inputResourcePath);
-
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- int tweetlinks = 0;
- int retweetlinks = 0;
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
-
- LOGGER.info("raw: {}", line);
-
- Class detected = new TwitterDocumentClassifier().detectClasses(line).get(0);
-
- ObjectNode event = (ObjectNode) mapper.readTree(line);
-
- assertThat(event, is(not(nullValue())));
-
- if ( detected == Tweet.class ) {
-
- Tweet tweet = mapper.convertValue(event, Tweet.class);
-
- assertThat(tweet, is(not(nullValue())));
- assertThat(tweet.getCreatedAt(), is(not(nullValue())));
- assertThat(tweet.getText(), is(not(nullValue())));
- assertThat(tweet.getUser(), is(not(nullValue())));
-
- tweetlinks += Optional.ofNullable(tweet.getEntities().getUrls().size()).orElse(0);
-
- } else if ( detected == Retweet.class ) {
-
- Retweet retweet = mapper.convertValue(event, Retweet.class);
-
- assertThat(retweet.getRetweetedStatus(), is(not(nullValue())));
- assertThat(retweet.getRetweetedStatus().getCreatedAt(), is(not(nullValue())));
- assertThat(retweet.getRetweetedStatus().getText(), is(not(nullValue())));
- assertThat(retweet.getRetweetedStatus().getUser(), is(not(nullValue())));
- assertThat(retweet.getRetweetedStatus().getUser().getId(), is(not(nullValue())));
- assertThat(retweet.getRetweetedStatus().getUser().getCreatedAt(), is(not(nullValue())));
-
- retweetlinks += Optional.ofNullable(retweet.getRetweetedStatus().getEntities().getUrls().size()).orElse(0);
-
- } else if ( detected == Delete.class ) {
-
- Delete delete = mapper.convertValue(event, Delete.class);
-
- assertThat(delete.getDelete(), is(not(nullValue())));
- assertThat(delete.getDelete().getStatus(), is(not(nullValue())));
- assertThat(delete.getDelete().getStatus().getId(), is(not(nullValue())));
- assertThat(delete.getDelete().getStatus().getUserId(), is(not(nullValue())));
-
- } else {
- Assert.fail();
- }
-
- }
- }
- } catch ( Exception ex ) {
- LOGGER.error("Exception: ", ex);
- Assert.fail();
- }
-
- assertThat(tweetlinks, is(greaterThan(0)));
- assertThat(retweetlinks, is(greaterThan(0)));
-
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/providers/TwitterFollowingProviderIT.java b/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/providers/TwitterFollowingProviderIT.java
deleted file mode 100644
index 6111b40..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/providers/TwitterFollowingProviderIT.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.test.providers;
-
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.twitter.provider.TwitterFollowingProvider;
-
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigValue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-@Test(dependsOnGroups = {"Account"}, groups = {"Providers"})
-public class TwitterFollowingProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterFollowingProviderIT.class);
-
- /**
- * Data Provider for TwitterFollowingProviderIT
- * [][0] = endpoint
- * [][1] = ids_only
- * [][2] = max_items
- * @return
- */
- @DataProvider(name = "TwitterFollowingProviderIT")
- public static Object[][] credentials() {
- return new Object[][] {
- {"followers", Boolean.FALSE, 40},
- {"followers", Boolean.TRUE, 10000},
- {"friends", Boolean.FALSE, 40 },
- {"friends", Boolean.TRUE, 100}
- };
- }
-
- @Test(dataProvider = "TwitterFollowingProviderIT")
- public void testTwitterFollowingProvider(String endpoint, Boolean ids_only, Integer max_items) throws Exception {
-
- String configfile = "./target/test-classes/TwitterFollowingProviderIT.conf";
- String outfile = "./target/test-classes/TwitterFollowingProviderIT-"+endpoint+"-"+ids_only+".stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- File conf = new File(configfile);
- Assert.assertTrue (conf.exists());
- Assert.assertTrue (conf.canRead());
- Assert.assertTrue (conf.isFile());
-
- System.setProperty("ENDPOINT", endpoint);
- System.setProperty("IDS_ONLY", Boolean.toString(ids_only));
- System.setProperty("MAX_ITEMS", Integer.toString(max_items));
- ConfigFactory.invalidateCaches();
- StreamsConfigurator.setConfig(ConfigFactory.load());
-
- Assert.assertTrue(ConfigFactory.parseFileAnySyntax(conf).withFallback(StreamsConfigurator.getConfig()).resolve().isResolved());
-
- Thread testThread = new Thread(() -> {
- try {
- TwitterFollowingProvider.main(args);
- } catch ( Exception ex ) {
- LOGGER.error("Test Exception!", ex);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- File out = new File(outfile);
- Assert.assertTrue (out.exists());
- Assert.assertTrue (out.canRead());
- Assert.assertTrue (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- Assert.assertEquals (outCounter.getLineNumber(), max_items.intValue());
-
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/providers/TwitterStreamProviderIT.java b/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/providers/TwitterStreamProviderIT.java
deleted file mode 100644
index 61d2d8e..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/providers/TwitterStreamProviderIT.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.test.providers;
-
-import org.apache.streams.twitter.provider.TwitterStreamProvider;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.testng.Assert.assertTrue;
-
-@Test(dependsOnGroups = {"Account"}, groups = {"Providers"})
-public class TwitterStreamProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterStreamProviderIT.class);
-
- final String outfile = "./target/test-classes/TwitterStreamProviderIT.stdout.txt";
- final String configfile = "./target/test-classes/TwitterStreamProviderIT.conf";
-
- @Test(groups = "TwitterStreamProviderIT")
- public void testTwitterStreamProvider() throws Exception {
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- TwitterStreamProvider.main(args);
- } catch ( Exception ex ) {
- LOGGER.error("Test Exception!", ex);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- File out = new File(outfile);
- assertTrue (out.exists());
- assertTrue (out.canRead());
- assertTrue (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- assertThat (outCounter.getLineNumber(), greaterThanOrEqualTo(1));
-
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/providers/TwitterTimelineProviderIT.java b/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/providers/TwitterTimelineProviderIT.java
deleted file mode 100644
index 89a406f..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/providers/TwitterTimelineProviderIT.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.test.providers;
-
-import org.apache.streams.twitter.provider.TwitterTimelineProvider;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-@Test(dependsOnGroups = {"Account"}, groups = {"Providers"})
-public class TwitterTimelineProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterTimelineProviderIT.class);
-
- @Test(groups = "TwitterTimelineProviderIT")
- public void testTwitterTimelineProvider() throws Exception {
-
- String configfile = "./target/test-classes/TwitterTimelineProviderIT.conf";
- String outfile = "./target/test-classes/TwitterTimelineProviderIT.stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- TwitterTimelineProvider.main(args);
- } catch ( Exception ex ) {
- LOGGER.error("Test Exception!", ex);
- }
- });
- testThread.start();
- testThread.join(600000);
-
- File out = new File(outfile);
- Assert.assertTrue (out.exists());
- Assert.assertTrue (out.canRead());
- Assert.assertTrue (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- Assert.assertEquals (outCounter.getLineNumber(), 1000);
-
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/providers/TwitterUserInformationProviderIT.java b/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/providers/TwitterUserInformationProviderIT.java
deleted file mode 100644
index 5e22203..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/providers/TwitterUserInformationProviderIT.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.test.providers;
-
-import org.apache.streams.twitter.provider.TwitterUserInformationProvider;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.greaterThan;
-
-@Test(dependsOnGroups = {"Account"}, groups = {"Providers"})
-public class TwitterUserInformationProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterUserInformationProviderIT.class);
-
- @Test
- public void testTwitterUserInformationProvider() throws Exception {
-
- String configfile = "./target/test-classes/TwitterUserInformationProviderIT.conf";
- String outfile = "./target/test-classes/TwitterUserInformationProviderIT.stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- TwitterUserInformationProvider.main(args);
- } catch ( Exception ex ) {
- LOGGER.error("Test Exception!", ex);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- File out = new File(outfile);
- Assert.assertTrue (out.exists());
- Assert.assertTrue (out.canRead());
- Assert.assertTrue (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- assertThat (outCounter.getLineNumber(), greaterThan(750));
-
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/utils/TwitterActivityConvertersTest.java b/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/utils/TwitterActivityConvertersTest.java
deleted file mode 100644
index 7bfb8ce..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/utils/TwitterActivityConvertersTest.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.test.utils;
-
-import org.apache.streams.converter.ActivityConverterUtil;
-import org.apache.streams.data.util.ActivityUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.twitter.converter.TwitterDateTimeFormat;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Follow;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.Tweet;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests {org.apache.streams.twitter.converter.*}
- */
-public class TwitterActivityConvertersTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterActivityConvertersTest.class);
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT).collect(Collectors.toList()));
-
- private ActivityConverterUtil activityConverterUtil = ActivityConverterUtil.getInstance();
-
- private String tweetJson = "{\"created_at\":\"Wed Dec 11 22:27:34 +0000 2013\",\"id\":12345,\"id_str\":\"12345\",\"text\":\"text\",\"source\":\"source\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":91407775,\"id_str\":\"12345\",\"name\":\"name\",\"screen_name\":\"screen_name\",\"location\":\"\",\"url\":null,\"description\":null,\"prote [...]
- private String retweetJson = "{\"created_at\":\"Wed Dec 11 22:27:34 +0000 2013\",\"id\":23456,\"id_str\":\"23456\",\"text\":\"text\",\"source\":\"web\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":163149656,\"id_str\":\"34567\",\"name\":\"name\",\"screen_name\":\"screen_name\",\"location\":\"location\",\"url\":\"http:\\/\\/www.youtube. [...]
-
- @Test
- public void testConvertTweet() throws Exception {
- Tweet tweet = mapper.readValue(tweetJson, Tweet.class);
- List<Activity> activityList = activityConverterUtil.convert(tweet);
- Assert.assertTrue(activityList.size() == 1);
- Activity activity = activityList.get(0);
- if ( !ActivityUtil.isValid(activity) ) {
- Assert.fail();
- }
- }
-
- @Test
- public void testConvertRetweet() throws Exception {
- Retweet retweet = mapper.readValue(retweetJson, Retweet.class);
- List<Activity> activityList = activityConverterUtil.convert(retweet);
- Assert.assertEquals(activityList.size(), 1);
- Activity activity = activityList.get(0);
- if ( !ActivityUtil.isValid(activity) ) {
- Assert.fail();
- }
- }
-
- @Test
- public void testConvertDelete() throws Exception {
- Delete delete = mapper.readValue(retweetJson, Delete.class);
- List<Activity> activityList = activityConverterUtil.convert(delete);
- Assert.assertEquals(activityList.size(), 1);
- Activity activity = activityList.get(0);
- if ( !ActivityUtil.isValid(activity) ) {
- Assert.fail();
- }
- }
-
- @Test
- public void testConvertFollow() throws Exception {
- Follow follow = mapper.readValue(retweetJson, Follow.class);
- List<Activity> activityList = activityConverterUtil.convert(follow);
- Assert.assertTrue(activityList.size() == 1);
- Activity activity = activityList.get(0);
- if ( !ActivityUtil.isValid(activity) ) {
- Assert.fail();
- }
- }
-
- @Test
- public void testConvertTweetString() {
- List<Activity> activityList = activityConverterUtil.convert(tweetJson);
- Assert.assertTrue(activityList.size() == 1);
- Activity activity = activityList.get(0);
- if ( !ActivityUtil.isValid(activity) ) {
- Assert.fail();
- }
- }
-
- @Test
- public void testConvertRetweetString() {
- List<Activity> activityList = activityConverterUtil.convert(retweetJson);
- Assert.assertTrue(activityList.size() == 1);
- Activity activity = activityList.get(0);
- if ( !ActivityUtil.isValid(activity) ) {
- Assert.fail();
- }
- }
-
- @Test
- public void testConvertDeleteString() {
- String deleteJson = "{\"delete\":{\"status\":{\"id\":56789,\"user_id\":67890,\"id_str\":\"56789\",\"user_id_str\":\"67890\"}}}\n";
- List<Activity> activityList = activityConverterUtil.convert(deleteJson);
- Assert.assertTrue(activityList.size() == 1);
- Activity activity = activityList.get(0);
- if ( !ActivityUtil.isValid(activity) ) {
- Assert.fail();
- }
- }
-
- @Test
- public void testConvertFollowString() {
- String followJson = "{\"follower\":{\"id\":12345},\"followee\":{\"id\":56789}}\n";
- List<Activity> activityList = activityConverterUtil.convert(followJson);
- Assert.assertTrue(activityList.size() == 1);
- Activity activity = activityList.get(0);
- if ( !ActivityUtil.isValid(activity) ) {
- Assert.fail();
- }
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/utils/TwitterActivityObjectsConvertersTest.java b/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/utils/TwitterActivityObjectsConvertersTest.java
deleted file mode 100644
index 6528441..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/utils/TwitterActivityObjectsConvertersTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.test.utils;
-
-import org.apache.streams.converter.ActivityObjectConverterProcessorConfiguration;
-import org.apache.streams.converter.ActivityObjectConverterUtil;
-import org.apache.streams.data.util.ActivityUtil;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.twitter.converter.TwitterDateTimeFormat;
-import org.apache.streams.twitter.converter.TwitterDocumentClassifier;
-import org.apache.streams.twitter.converter.TwitterJsonUserActivityObjectConverter;
-import org.apache.streams.twitter.pojo.User;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests {org.apache.streams.twitter.converter.*}
- */
-public class TwitterActivityObjectsConvertersTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TwitterActivityObjectsConvertersTest.class);
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance(Stream.of(TwitterDateTimeFormat.TWITTER_FORMAT)
- .collect(Collectors.toList()));
-
- private ActivityObjectConverterProcessorConfiguration activityObjectConverterProcessorConfiguration =
- new ActivityObjectConverterProcessorConfiguration()
- .withClassifiers(Stream.of(new TwitterDocumentClassifier()).collect(Collectors.toList()))
- .withConverters(Stream.of(new TwitterJsonUserActivityObjectConverter()).collect(Collectors.toList()));
-
- private ActivityObjectConverterUtil activityObjectConverterUtil = ActivityObjectConverterUtil.getInstance(activityObjectConverterProcessorConfiguration);
-
- private String userJson = "{\"id\":1663018644,\"id_str\":\"1663018644\",\"name\":\"M.R. Clark\",\"screen_name\":\"cantennisfan\",\"location\":\"\",\"url\":null,\"description\":null,\"protected\":false,\"verified\":false,\"followers_count\":0,\"friends_count\":5,\"listed_count\":0,\"favourites_count\":2,\"statuses_count\":72,\"created_at\":\"Sun Aug 11 17:23:47 +0000 2013\",\"utc_offset\":-18000,\"time_zone\":\"Eastern Time (US & Canada)\",\"geo_enabled\":false,\"lang\":\"en\",\"contrib [...]
-
- @Test
- public void testConvertUser() throws Exception {
- User user = mapper.readValue(userJson, User.class);
- ActivityObject activityObject = activityObjectConverterUtil.convert(user);
- assert ( activityObject != null );
- if ( !ActivityUtil.isValid(activityObject) ) {
- Assert.fail();
- }
- }
-
- @Test
- public void testConvertUserString() {
- ActivityObject activityObject = activityObjectConverterUtil.convert(userJson);
- assert ( activityObject != null );
- if ( !ActivityUtil.isValid(activityObject) ) {
- Assert.fail();
- }
- }
-}
diff --git a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/utils/TwitterDocumentClassifierTest.java b/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/utils/TwitterDocumentClassifierTest.java
deleted file mode 100644
index 418491a..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/java/org/apache/streams/twitter/test/utils/TwitterDocumentClassifierTest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.twitter.test.utils;
-
-import org.apache.streams.twitter.converter.TwitterDocumentClassifier;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Follow;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.Tweet;
-import org.apache.streams.twitter.pojo.User;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.List;
-
-/**
- * Tests {@link: org.apache.streams.twitter.processor.TwitterEventClassifier}
- */
-public class TwitterDocumentClassifierTest {
-
- private String tweet = "{\"created_at\":\"Wed Dec 11 22:27:34 +0000 2013\",\"id\":12345,\"id_str\":\"12345\",\"text\":\"text\",\"source\":\"source\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":91407775,\"id_str\":\"12345\",\"name\":\"name\",\"screen_name\":\"screen_name\",\"location\":\"\",\"url\":null,\"description\":null,\"protected [...]
- private String retweet = "{\"created_at\":\"Wed Dec 11 22:27:34 +0000 2013\",\"id\":23456,\"id_str\":\"23456\",\"text\":\"text\",\"source\":\"web\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":163149656,\"id_str\":\"34567\",\"name\":\"name\",\"screen_name\":\"screen_name\",\"location\":\"location\",\"url\":\"http:\\/\\/www.youtube.com\ [...]
- private String delete = "{\"delete\":{\"status\":{\"id\":56789,\"user_id\":67890,\"id_str\":\"56789\",\"user_id_str\":\"67890\"}}}\n";
- private String follow = "{\"follower\":{\"id\":12345},\"followee\":{\"id\":56789}}\n";
- private String user = "{\"location\":\"\",\"default_profile\":true,\"profile_background_tile\":false,\"statuses_count\":1,\"lang\":\"en\",\"profile_link_color\":\"0084B4\",\"id\":67890,\"following\":false,\"protected\":false,\"favourites_count\":0,\"profile_text_color\":\"333333\",\"description\":\"\",\"verified\":false,\"contributors_enabled\":false,\"profile_sidebar_border_color\":\"C0DEED\",\"name\":\"name\",\"profile_background_color\":\"C0DEED\",\"created_at\":\"Fri Apr 17 12:35:5 [...]
-
- @Test
- public void testDetectTweet() {
- List<Class> detected = new TwitterDocumentClassifier().detectClasses(tweet);
- Assert.assertTrue(detected.size() == 1);
- Class result = detected.get(0);
- if ( !result.equals(Tweet.class) ) {
- Assert.fail();
- }
- }
-
- @Test
- public void testDetectRetweet() {
- List<Class> detected = new TwitterDocumentClassifier().detectClasses(retweet);
- Assert.assertTrue(detected.size() == 1);
- Class result = detected.get(0);
- if ( !result.equals(Retweet.class) ) {
- Assert.fail();
- }
- }
-
- @Test
- public void testDetectDelete() {
- List<Class> detected = new TwitterDocumentClassifier().detectClasses(delete);
- Assert.assertTrue(detected.size() == 1);
- Class result = detected.get(0);
- if ( !result.equals(Delete.class) ) {
- Assert.fail();
- }
- }
-
- @Test
- public void testDetectFollow() {
- List<Class> detected = new TwitterDocumentClassifier().detectClasses(follow);
- Assert.assertTrue(detected.size() == 1);
- Class result = detected.get(0);
- if ( !result.equals(Follow.class) ) {
- Assert.fail();
- }
- }
-
- @Test
- public void testDetectUser() {
- List<Class> detected = new TwitterDocumentClassifier().detectClasses(user);
- Assert.assertTrue(detected.size() == 1);
- Class result = detected.get(0);
- if (!result.equals(User.class)) {
- Assert.fail();
- }
- }
-
-}
diff --git a/streams-contrib/streams-provider-twitter/src/test/resources/TwitterFollowingProviderIT.conf b/streams-contrib/streams-provider-twitter/src/test/resources/TwitterFollowingProviderIT.conf
deleted file mode 100644
index e9930be..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/resources/TwitterFollowingProviderIT.conf
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-twitter {
- info = [
- 18055613
- ]
- endpoint = ${ENDPOINT}
- ids_only = ${IDS_ONLY}
- max_items = ${MAX_ITEMS}
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/test/resources/TwitterIT.conf b/streams-contrib/streams-provider-twitter/src/test/resources/TwitterIT.conf
deleted file mode 100644
index af88ba9..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/resources/TwitterIT.conf
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-twitter {
- retrySleepMs = 100
- retryMax = 1
-}
diff --git a/streams-contrib/streams-provider-twitter/src/test/resources/TwitterObjectMapperIT.conf b/streams-contrib/streams-provider-twitter/src/test/resources/TwitterObjectMapperIT.conf
deleted file mode 100644
index aa55167..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/resources/TwitterObjectMapperIT.conf
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-inputResourcePath = "/TwitterStreamProviderIT.stdout.txt"
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-twitter/src/test/resources/TwitterStreamProviderIT.conf b/streams-contrib/streams-provider-twitter/src/test/resources/TwitterStreamProviderIT.conf
deleted file mode 100644
index 4f808eb..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/resources/TwitterStreamProviderIT.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-twitter {
- endpoint = sample
- track = [
- "data"
- ]
-}
diff --git a/streams-contrib/streams-provider-twitter/src/test/resources/TwitterTimelineProviderIT.conf b/streams-contrib/streams-provider-twitter/src/test/resources/TwitterTimelineProviderIT.conf
deleted file mode 100644
index 779a1bf..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/resources/TwitterTimelineProviderIT.conf
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-twitter.info = [
- "TheASF"
-]
-twitter.max_items = 1000
diff --git a/streams-contrib/streams-provider-twitter/src/test/resources/TwitterUserInformationProviderIT.conf b/streams-contrib/streams-provider-twitter/src/test/resources/TwitterUserInformationProviderIT.conf
deleted file mode 100644
index f6949cf..0000000
--- a/streams-contrib/streams-provider-twitter/src/test/resources/TwitterUserInformationProviderIT.conf
+++ /dev/null
@@ -1,1018 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-twitter.info = [
- 3424266646
- 3277467241
- 3244517214
- 29953647
- 63818319
- 1528436754
- 405580894
- 322778026
- 172382176
- 633076833
- 703735608
- 2347223440
- 2907929487
- 950240089
- 1418546592
- 3318418717
- 2848958704
- 1120797264
- 933623324
- 2977700375
- 328204518
- 585131136
- 2868789793
- 158347647
- 2915413161
- 2217367263
- 2534019247
- 3033565239
- 377379801
- 2525341814
- 3123827524
- 1840932523
- 3307643975
- 3301777832
- 961987748
- 3205632255
- 2799469322
- 17730681
- 1495242662
- 1909516123
- 263933760
- 312651511
- 2479527469
- 2357151036
- 346433828
- 44801893
- 1049697306
- 2779673194
- 18323141
- 2172488902
- 2373431930
- 1038322550
- 2946211549
- 2911057543
- 1186036284
- 2878076317
- 1312950464
- 57323685
- 32929857
- 301933631
- 2852217152
- 330422649
- 98470876
- 933125156
- 3237125761
- 914882005
- 1560239652
- 900444860
- 402918702
- 1820690166
- 3074359086
- 353183684
- 528544881
- 1881638161
- 2751762993
- 3161315692
- 3305680079
- 1721613488
- 513068659
- 627186234
- 3203648416
- 1541163325
- 1882043502
- 29071727
- 610104090
- 2819781014
- 2909115204
- 213886397
- 3249385591
- 3086875073
- 87040031
- 2202487475
- 334896132
- 49163181
- 3433984816
- 543969362
- 489445461
- 855051894
- 2792040175
- 117051455
- 438599410
- 1387329846
- 711595782
- 3230662766
- 2766672269
- 2926781875
- 863203928
- 517199566
- 201645935
- 1555939147
- 2943152669
- 1324775431
- 400234897
- 2347416842
- 1558112510
- 474415350
- 2153710970
- 1408335014
- 3633713483
- 3166021013
- 3530993294
- 332598229
- 308252069
- 3317826986
- 572175644
- 1718271572
- 2869090090
- 23725109
- 1926137280
- 1486830500
- 743080386
- 3250479720
- 2560441544
- 2715649872
- 287089153
- 18761334
- 2305577745
- 724860668
- 193306049
- 2615761979
- 2463299598
- 1436916012
- 919019185
- 90502449
- 50689522
- 1383774679
- 612784850
- 410319975
- 833440153
- 442322844
- 2181167094
- 94012832
- 112748352
- 1474618075
- 158262669
- 2391506308
- 882502026
- 2693660146
- 2971933908
- 55271184
- 2287356556
- 2895756090
- 407147132
- 3262181
- 313317193
- 2729137002
- 2939122360
- 2751601568
- 1215082350
- 124866576
- 274292311
- 3310301042
- 95407473
- 24993769
- 1342908648
- 1805339413
- 3118252036
- 893269387
- 1481149014
- 463288019
- 75008083
- 2895489727
- 965493739
- 278637248
- 1937513246
- 422218268
- 3320995462
- 78682286
- 2777069098
- 2909553730
- 2914338670
- 1251667531
- 2764034755
- 532659717
- 269002510
- 29373713
- 358075450
- 633880614
- 200374379
- 141628294
- 1513028977
- 116798791
- 2937455354
- 246194623
- 793925970
- 115594167
- 82463176
- 324774974
- 185844856
- 2462295999
- 3555105016
- 1029169117
- 2689309484
- 1587145976
- 1607241271
- 3032276402
- 183916933
- 63766245
- 151217255
- 2781098109
- 252081559
- 1608788256
- 41984573
- 1896587353
- 40136999
- 295505814
- 384867933
- 116947371
- 255703939
- 2687800732
- 76543916
- 881649782
- 2765729924
- 1715695669
- 1965383022
- 2888214228
- 21820514
- 1727966414
- 2581992818
- 103999565
- 741018846
- 446792386
- 2568989424
- 2780674777
- 465934916
- 3378294885
- 2885604327
- 3336273419
- 130742941
- 2327629099
- 1103818104
- 3050036073
- 2882456842
- 2702914248
- 2153674818
- 132825659
- 289758699
- 2995946100
- 3027449217
- 2708029160
- 1529367002
- 608170333
- 140446819
- 2790688993
- 1597308192
- 14462028
- 104062608
- 370274893
- 356145607
- 566542629
- 112587243
- 39372070
- 146853060
- 2440984657
- 3074554539
- 204701034
- 887623447
- 1971521630
- 2457208175
- 466113358
- 1574643830
- 1465533884
- 2500404589
- 1633154150
- 1349117870
- 1658071267
- 593022891
- 3094177813
- 1304672510
- 3385525697
- 2916225552
- 2759773715
- 1369215552
- 1058390078
- 2532850321
- 351483656
- 1902796704
- 113000738
- 2241245557
- 2416606754
- 408729540
- 2530294556
- 2936808249
- 3138999692
- 2679987883
- 1448537377
- 2524773906
- 942079406
- 2217584389
- 3059427504
- 3028507725
- 632766658
- 3302663431
- 2914832897
- 93487101
- 2786054379
- 1339647769
- 531402307
- 402066474
- 337936675
- 2760568625
- 1385916396
- 2595560922
- 421910477
- 1713100813
- 352016040
- 415247994
- 1883606209
- 2974994111
- 1118022211
- 3096979637
- 711889867
- 262890561
- 233810062
- 1877177168
- 964106670
- 164985413
- 2920420361
- 318936782
- 3289826764
- 145873735
- 2523059919
- 2409896179
- 2292047201
- 285674825
- 2765549780
- 2359541905
- 2419103894
- 358884588
- 206231205
- 136500778
- 1397885138
- 2625422097
- 2524578002
- 604278657
- 2625634867
- 73168019
- 407448958
- 189276174
- 2507896925
- 80880449
- 520177827
- 418469102
- 2925075456
- 615730636
- 2995998941
- 2697270934
- 497135011
- 2944598402
- 428706893
- 1345291712
- 388751708
- 130092079
- 2984741882
- 1047514436
- 15927135
- 2884357840
- 294362779
- 2870985800
- 1720400449
- 130027314
- 2970518577
- 240923858
- 1613498838
- 708321211
- 1403382426
- 2602186970
- 1596855998
- 280062526
- 2716454552
- 268720451
- 2869044811
- 1911762488
- 392373280
- 2151082712
- 2770919004
- 231541900
- 60122778
- 390006102
- 240167506
- 1558314660
- 221608257
- 852829933
- 461669243
- 239778483
- 502146157
- 1471963970
- 276426707
- 2336546150
- 323595235
- 128670043
- 1308641714
- 1411112756
- 3011727217
- 3082006921
- 450537474
- 2673101407
- 2416030447
- 51952627
- 708057486
- 833620748
- 3024957797
- 2147572362
- 1712467098
- 2899300501
- 1348351772
- 2923114629
- 2779232814
- 21306308
- 1466314507
- 1224588289
- 81307783
- 42717316
- 315972617
- 434649827
- 105839296
- 366063496
- 34045892
- 3076447389
- 92437198
- 3124335006
- 1444393410
- 351737762
- 1919360383
- 2836048345
- 1670939112
- 722140159
- 92939425
- 2932728756
- 2831872033
- 1354255123
- 1689738186
- 463578260
- 2881582438
- 912252510
- 3226221887
- 390827200
- 269169237
- 1450007192
- 2735984326
- 3029836305
- 28291382
- 785668627
- 567287970
- 1480004420
- 131927864
- 2958631308
- 488490020
- 2603422688
- 3186614985
- 177373618
- 2466506329
- 2651294251
- 3367170684
- 2673870882
- 369098635
- 242011326
- 18099277
- 1922210574
- 3093762445
- 470634878
- 1674607392
- 2920526283
- 3261677580
- 2192187078
- 485599960
- 1854850729
- 95198467
- 2228217740
- 2171528344
- 2957461230
- 226615737
- 1624183567
- 158597677
- 2909224690
- 19278114
- 2488284258
- 2777071149
- 1598064697
- 2740691127
- 3100908480
- 1147010126
- 2741161553
- 439971668
- 3247227273
- 2884261062
- 3127250575
- 2942021278
- 539428196
- 409599986
- 3161801331
- 2328613860
- 1903013437
- 313082004
- 2580495721
- 209464435
- 600172085
- 339541217
- 62219810
- 583287316
- 295891933
- 561683767
- 229192352
- 1357869918
- 235438136
- 1599249169
- 583879210
- 507744802
- 1696336261
- 2323537206
- 36882220
- 541528426
- 956202559
- 387936537
- 211658842
- 2685186010
- 2581656488
- 391154378
- 122932105
- 409764153
- 129737967
- 2848806360
- 3054860719
- 372199585
- 2316121597
- 703345746
- 3335505287
- 2466151422
- 380038166
- 420561214
- 2977085351
- 110955327
- 3004295886
- 2362857361
- 3053844460
- 3182081552
- 324208260
- 2571790321
- 1061498868
- 2187395299
- 2187482779
- 3096652530
- 2538239672
- 3809634552
- 2306848839
- 1544061547
- 151075965
- 3250238556
- 16157689
- 1692663644
- 1356000732
- 436774994
- 45503055
- 1086037316
- 2798297775
- 2923485772
- 58731726
- 211816170
- 885013716
- 2608529078
- 2954917057
- 2271021600
- 173743066
- 451543575
- 3219728436
- 399824828
- 2464688153
- 2541069631
- 1522892262
- 3167829845
- 944851321
- 2471474509
- 68073858
- 1496221376
- 13979882
- 2218792189
- 302123873
- 2845915546
- 431402814
- 1364254945
- 2711277666
- 2766696876
- 2495441323
- 2844317433
- 138009079
- 2578631100
- 478167529
- 1222728360
- 1323688411
- 2883066187
- 2443554697
- 411631689
- 68537682
- 1027019269
- 1660752493
- 987324488
- 2764106926
- 2184511674
- 103419315
- 2310456424
- 1572938088
- 2554895281
- 34138105
- 2942100621
- 160517898
- 285075974
- 2260805169
- 19390498
- 301696842
- 2588239985
- 2886588596
- 2962622367
- 1867897483
- 2827053488
- 1447767319
- 2924491293
- 167327096
- 3309592402
- 2795575638
- 578758971
- 2888665561
- 30542348
- 1437049609
- 2242541566
- 74354017
- 58900854
- 2159055031
- 246517688
- 2916873012
- 1110055280
- 562430843
- 761797794
- 1648208552
- 301483343
- 2896842048
- 522103295
- 1578517986
- 2659610776
- 2890560429
- 1427665578
- 268363160
- 563709041
- 2172300002
- 2791262431
- 3039809351
- 2914940301
- 2746560353
- 2892191616
- 71596845
- 233770184
- 1530949130
- 105906110
- 755347622
- 490836906
- 357603454
- 324517203
- 2835402315
- 3285479894
- 86368327
- 238219970
- 3153173945
- 2732361234
- 2357626327
- 346602505
- 13732632
- 44055265
- 2998032219
- 482072312
- 1721073866
- 1386781034
- 168194206
- 1213443144
- 181296114
- 942598400
- 2955577216
- 582056669
- 747540468
- 2371722140
- 360824004
- 3023711736
- 207032580
- 2748107976
- 464428175
- 3150849096
- 85450014
- 2840066340
- 2287819200
- 240931426
- 553606800
- 397876544
- 2195298230
- 2601812005
- 3013344739
- 17599363
- 1572639314
- 3377673407
- 303420278
- 2811879995
- 526860891
- 346333874
- 113568311
- 705488304
- 3238867619
- 333772149
- 373309716
- 300472003
- 3223424681
- 2895699896
- 3241119570
- 1147453440
- 3135402609
- 521763744
- 2702966971
- 2878317616
- 845031697
- 2855454471
- 3051902539
- 482306439
- 129173738
- 306572138
- 2941951538
- 762707233
- 2732608168
- 1228456939
- 246020724
- 1920607602
- 14434245
- 1254943537
- 1520746602
- 150745124
- 1350160351
- 38707222
- 267766858
- 2992121760
- 712666764
- 983036864
- 289490939
- 269797384
- 100215048
- 3099557245
- 2339741570
- 306005146
- 1182227460
- 288235870
- 1412832260
- 455190443
- 489912183
- 448994061
- 2944595072
- 2453094914
- 2899434206
- 59288818
- 2824706688
- 423363992
- 972850482
- 997868714
- 1203750733
- 176147179
- 115110596
- 2978397615
- 2528946267
- 620180433
- 365949935
- 110609853
- 1533494268
- 2723839166
- 34186887
- 2864430424
- 76942977
- 361086733
- 2724200587
- 635206139
- 2757801421
- 19651443
- 3364322949
- 2770576744
- 2168612560
- 764020297
- 2558268513
- 2855384901
- 1881414907
- 2502212139
- 3250037586
- 2525185944
- 591375982
- 707911211
- 3025041666
- 19785599
- 2311172950
- 922817815
- 739363530
- 2812894393
- 2496283986
- 206162815
- 590916342
- 354053245
- 2735195854
- 2788759128
- 3510947235
- 3490740532
- 2920847304
- 2681444558
- 2856805755
- 3103899682
- 145893832
- 3065663910
- 2736009516
- 2835226230
- 1590913771
- 2700889555
- 2221272164
- 109780161
- 700221218
- 541753453
- 126575915
- 274336817
- 2498172455
- 2809515630
- 2588774684
- 296734891
- 2212410182
- 243027454
- 1336526904
- 397062736
- 449331876
- 30619307
- 2310483811
- 2437586509
- 191710730
- 1084185378
- 2831486681
- 1606477879
- 969600636
- 529783214
- 2928131586
- 190041293
- 2967031274
- 2165962781
- 376501355
- 284137985
- 266863824
- 407944074
- 108456036
- 1641294422
- 900733706
- 1063071450
- 1682722328
- 341419520
- 1644293778
- 2245151467
- 511176989
- 241922669
- 3388315624
- 1909431145
- 2223820028
- 600581315
- 1723555076
- 2748445313
- 561211823
- 561022931
- 2751429993
- 2714908343
- 16165257
- 524623359
- 306741266
- 469994381
- 2561892084
- 998802661
- 1492924374
- 789039140
- 210150093
- 817544820
- 35740178
- 326162841
- 1447331628
- 17493441
- 2874693608
- 965027312
- 261936985
- 510564259
- 728031187
- 164696234
- 2204519310
- 1626241164
- 1024940588
- 221486613
- 571084565
- 3029264508
- 221716563
- 2211417135
- 499972359
- 1565989165
- 2436927208
- 381029291
- 2730580620
- 3436438413
- 2466014604
- 538990742
- 2935470687
- 1162845468
- 468108082
- 2383897542
- 2542119658
- 1962281514
- 171235080
- 536915535100125185
- 2841076618
- 3006098500
- 1057158554
- 3245676721
- 251087536
- 3082811549
- 281785349
- 1674871100
- 1898659951
- 1414854156
- 428693618
- 2385953101
- 2281213477
- 2786368894
- 2253203998
- 357277727
- 1358707970
- 545186198
- 3033613587
- 107121821
- 595965259
- 583894637
- 1306698787
- 442262869
- 2868353318
- 1908436844
- 271982042
- 495202171
- 251586884
- 3151032974
- 2213682568
- 1203133039
- 193128957
- 597407120
- 2781102086
- 369254505
- 62831036
- 2328734640
- 2579064082
- 3271313827
- 2880366619
- 2323026113
- 446380518
- 245418139
- 261211664
- 1893329208
- 3406596309
- 584967077
- 1708862304
- 388961426
- 2421535351
- 2194375668
- 2790313673
- 2728894977
- 2829174824
- 784541196
- 959902393
- 249705367
- 1677679309
- 2825975175
- 1305768366
- 373475046
- 785362464
- 419607671
- 61031675
- 3854236343
- 714603248
- 1301447720
- 827660912
- 2383764684
- 3180084906
- 3265558124
- 608536922
- 238943561
-]
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-youtube/README.md b/streams-contrib/streams-provider-youtube/README.md
deleted file mode 100644
index ef879ce..0000000
--- a/streams-contrib/streams-provider-youtube/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-provider-youtube
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-contrib/streams-provider-youtube/pom.xml b/streams-contrib/streams-provider-youtube/pom.xml
deleted file mode 100644
index 48f61a2..0000000
--- a/streams-contrib/streams-provider-youtube/pom.xml
+++ /dev/null
@@ -1,184 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/-->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-contrib</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-provider-youtube</artifactId>
-
- <properties>
- <youtube.client.version>v3-rev178-1.22.0</youtube.client.version>
- <google.client.version>1.22.0</google.client.version>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
- <repositories>
- <repository>
- <id>google-api-services</id>
- <url>http://google-api-client-libraries.appspot.com/mavenrepo</url>
- </repository>
- </repositories>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo-extensions</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.http-client</groupId>
- <artifactId>google-http-client-jackson2</artifactId>
- <version>${google.client.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.oauth-client</groupId>
- <artifactId>google-oauth-client-jetty</artifactId>
- <version>${google.client.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.google.collections</groupId>
- <artifactId>google-collections</artifactId>
- <version>1.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.google.api-client</groupId>
- <artifactId>google-api-client</artifactId>
- <version>${google.client.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.apis</groupId>
- <artifactId>google-api-services-youtube</artifactId>
- <version>${youtube.client.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-runtime-local</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.youtube.pojo</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <configuration>
- <skipTests>${skipITs}</skipTests>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/processor/YoutubeTypeConverter.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/processor/YoutubeTypeConverter.java
deleted file mode 100644
index ee9ca18..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/processor/YoutubeTypeConverter.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.youtube.serializer.YoutubeActivityUtil;
-import org.apache.streams.youtube.serializer.YoutubeChannelDeserializer;
-import org.apache.streams.youtube.serializer.YoutubeEventClassifier;
-import org.apache.streams.youtube.serializer.YoutubeVideoDeserializer;
-
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.youtube.model.Channel;
-import com.google.api.services.youtube.model.Video;
-import org.apache.commons.lang.NotImplementedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Queue;
-
-public class YoutubeTypeConverter implements StreamsProcessor {
-
- public static final String STREAMS_ID = "YoutubeTypeConverter";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeTypeConverter.class);
-
- private StreamsJacksonMapper mapper;
- private Queue<Video> inQueue;
- private Queue<StreamsDatum> outQueue;
- private int count = 0;
-
- public YoutubeTypeConverter() {}
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum streamsDatum) {
- StreamsDatum result = null;
-
- try {
- Object item = streamsDatum.getDocument();
-
- LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass());
- Activity activity;
-
- if (item instanceof String) {
- item = deserializeItem(item);
- }
-
- if (item instanceof Video) {
- activity = new Activity();
- YoutubeActivityUtil.updateActivity((Video)item, activity, streamsDatum.getId());
- } else if (item instanceof Channel) {
- activity = new Activity();
- YoutubeActivityUtil.updateActivity((Channel)item, activity, null);
- } else {
- throw new NotImplementedException("Type conversion not implement for type : " + item.getClass().getName());
- }
-
- if (activity != null) {
- result = new StreamsDatum(activity);
- count++;
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while converting Video to Activity: {}", ex);
- }
-
- if (result != null) {
- List<StreamsDatum> streamsDatumList = new ArrayList<>();
- streamsDatumList.add(result);
- return streamsDatumList;
- } else {
- return new ArrayList<>();
- }
- }
-
- private Object deserializeItem(Object item) {
- try {
- Class klass = YoutubeEventClassifier.detectClass((String) item);
- if (klass.equals(Video.class)) {
- item = mapper.readValue((String) item, Video.class);
- } else if (klass.equals(Channel.class)) {
- item = mapper.readValue((String) item, Channel.class);
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserializeItem: {}", ex);
- }
-
- return item;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- mapper = StreamsJacksonMapper.getInstance();
-
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Video.class, new YoutubeVideoDeserializer());
- mapper.registerModule(simpleModule);
- simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Channel.class, new YoutubeChannelDeserializer());
- mapper.registerModule(simpleModule);
- }
-
- @Override
- public void cleanUp() {}
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollector.java
deleted file mode 100644
index 69198df..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollector.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.youtube.YoutubeConfiguration;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.services.youtube.YouTube;
-import com.google.api.services.youtube.model.Channel;
-import com.google.gson.Gson;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-/**
- * Collects YoutubeChannelData on behalf of YoutubeChannelProvider.
- */
-public class YoutubeChannelDataCollector extends YoutubeDataCollector {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeChannelDataCollector.class);
- private static final String CONTENT = "snippet,contentDetails,statistics,topicDetails";
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
- private static final int MAX_ATTEMPTS = 5;
-
- private YouTube youTube;
- private BlockingQueue<StreamsDatum> queue;
- private BackOffStrategy strategy;
- private UserInfo userInfo;
- private YoutubeConfiguration youtubeConfig;
-
- /**
- * YoutubeChannelDataCollector constructor.
- * @param youTube YouTube
- * @param queue BlockingQueue of StreamsDatum
- * @param strategy BackOffStrategy
- * @param userInfo UserInfo
- * @param youtubeConfig YoutubeConfiguration
- */
- public YoutubeChannelDataCollector(
- YouTube youTube,
- BlockingQueue<StreamsDatum> queue,
- BackOffStrategy strategy,
- UserInfo userInfo,
- YoutubeConfiguration youtubeConfig) {
- this.youTube = youTube;
- this.queue = queue;
- this.strategy = strategy;
- this.userInfo = userInfo;
- this.youtubeConfig = youtubeConfig;
- }
-
- @Override
- public void run() {
- Gson gson = new Gson();
-
- int item_count = 0;
- int last_count = 0;
- int page_count = 0;
-
- try {
- int attempt = 0;
- YouTube.Channels.List channelLists = this.youTube.channels().list(CONTENT).setId(this.userInfo.getUserId()).setKey(this.youtubeConfig.getApiKey());
- boolean tryAgain = false;
- do {
- try {
- List<Channel> channels = channelLists.execute().getItems();
- last_count = channels.size();
- for (Channel channel : channels) {
- String json = gson.toJson(channel);
- this.queue.put(new StreamsDatum(json, channel.getId()));
- item_count++;
- }
- if (StringUtils.isEmpty(channelLists.getPageToken())) {
- channelLists = null;
- } else {
- channelLists = this.youTube.channels().list(CONTENT).setId(this.userInfo.getUserId()).setOauthToken(this.youtubeConfig.getApiKey())
- .setPageToken(channelLists.getPageToken());
- }
- page_count++;
- } catch (GoogleJsonResponseException gjre) {
- LOGGER.warn("GoogleJsonResposneException caught : {}", gjre);
- tryAgain = backoffAndIdentifyIfRetry(gjre, this.strategy);
- ++attempt;
- } catch (Throwable throwable) {
- LOGGER.warn("Unable to get channel info for id : {}", this.userInfo.getUserId());
- LOGGER.warn("Excpection thrown while trying to get channel info : {}", throwable);
- }
- }
- while ((tryAgain && attempt < MAX_ATTEMPTS) || channelLists != null);
-
- LOGGER.info("item_count: {} last_count: {} page_count: {} ", item_count, last_count, page_count);
-
- } catch (Throwable throwable) {
- LOGGER.warn(throwable.getMessage());
- }
- }
-
-
-}
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelProvider.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelProvider.java
deleted file mode 100644
index d8a09d5..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeChannelProvider.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.youtube.YoutubeConfiguration;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.services.youtube.YouTube;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Retrieve recent activity from a list of channels.
- */
-public class YoutubeChannelProvider extends YoutubeProvider {
-
- public YoutubeChannelProvider() {
- super();
- }
-
- public YoutubeChannelProvider(YoutubeConfiguration config) {
- super(config);
- }
-
- /**
- * To use from command line:
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- * <p/>
- * youtube.oauth.pathToP12KeyFile
- * youtube.oauth.serviceAccountEmailAddress
- * youtube.apiKey
- * youtube.youtubeUsers
- * <p/>
- * Launch using:
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.youtube.provider.YoutubeUserActivityProvider -Dexec.args="application.conf tweets.json"
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- YoutubeConfiguration config = new ComponentConfigurator<>(YoutubeConfiguration.class).detectConfiguration(typesafe, "youtube");
- YoutubeChannelProvider provider = new YoutubeChannelProvider(config);
-
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- for (StreamsDatum datum : provider.readCurrent()) {
- String json;
- try {
- if (datum.getDocument() instanceof String) {
- json = (String) datum.getDocument();
- } else {
- json = mapper.writeValueAsString(datum.getDocument());
- }
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while (provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo) {
- return new YoutubeChannelDataCollector(youtube, queue, strategy, userInfo, this.config);
- }
-}
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeDataCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeDataCollector.java
deleted file mode 100644
index b05365c..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeDataCollector.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.provider;
-
-import org.apache.streams.util.api.requests.backoff.BackOffException;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base Collector for Youtube Data.
- */
-public abstract class YoutubeDataCollector implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeDataCollector.class);
-
- /**
- * Looks at the status code of the expception. If the code indicates that the request should be retried,
- * it executes the back off strategy and returns true.
- * @param gjre
- * @param backOff
- * @return returns true if the error code of the exception indicates the request should be retried.
- */
- public boolean backoffAndIdentifyIfRetry(GoogleJsonResponseException gjre, BackOffStrategy backOff) throws BackOffException {
- boolean tryAgain = false;
-
- switch (gjre.getStatusCode()) {
- case 400 :
- LOGGER.warn("Bad Request : {}", gjre);
- break;
- case 401 :
- LOGGER.warn("Invalid Credentials : {}", gjre);
- break;
- case 403 :
- LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage());
- backOff.backOff();
- tryAgain = true;
- break;
- case 503 :
- LOGGER.warn("Google Backend Service Error : {}", gjre);
- break;
- default:
- LOGGER.warn("Google Service returned error : {}", gjre);
- tryAgain = true;
- backOff.backOff();
- break;
- }
-
- return tryAgain;
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeProvider.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeProvider.java
deleted file mode 100644
index da21722..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeProvider.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.ComponentUtils;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
-import org.apache.streams.youtube.YoutubeConfiguration;
-
-import com.google.api.client.auth.oauth2.Credential;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.http.javanet.NetHttpTransport;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.youtube.YouTube;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import org.apache.commons.lang3.StringUtils;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.math.BigInteger;
-import java.security.GeneralSecurityException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public abstract class YoutubeProvider implements StreamsProvider {
-
- private static final String STREAMS_ID = "YoutubeProvider";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeProvider.class);
- private static final int MAX_BATCH_SIZE = 1000;
- /**
- * Define a global instance of the HTTP transport.
- */
- private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
- /**
- * Define a global instance of the JSON factory.
- */
- private static final JsonFactory JSON_FACTORY = new JacksonFactory();
- private static final int DEFAULT_THREAD_POOL_SIZE = 5;
- protected YouTube youtube;
- protected YoutubeConfiguration config;
- // This OAuth 2.0 access scope allows for full read/write access to the
- // authenticated user's account.
- private List<String> scopes = Collections.singletonList("https://www.googleapis.com/auth/youtube");
- private List<ListenableFuture<Object>> futures = new ArrayList<>();
- private ListeningExecutorService executor;
- private BlockingQueue<StreamsDatum> datumQueue;
- private AtomicBoolean isComplete;
- private boolean previousPullWasEmpty;
-
- /**
- * YoutubeProvider constructor.
- * Resolves config from JVM 'youtube'.
- */
- public YoutubeProvider() {
- this.config = new ComponentConfigurator<>(YoutubeConfiguration.class)
- .detectConfiguration(StreamsConfigurator.getConfig().getConfig("youtube"));
-
- Objects.requireNonNull(this.config.getApiKey());
- }
-
- /**
- * YoutubeProvider constructor - uses supplied YoutubeConfiguration.
- * @param config YoutubeConfiguration
- */
- public YoutubeProvider(YoutubeConfiguration config) {
- this.config = config;
-
- Objects.requireNonNull(this.config.getApiKey());
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- try {
- this.youtube = createYouTubeClient();
- } catch (IOException | GeneralSecurityException ex) {
- LOGGER.error("Failed to created oauth for YouTube : {}", ex);
- throw new RuntimeException(ex);
- }
-
- this.executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(DEFAULT_THREAD_POOL_SIZE));
- this.datumQueue = new LinkedBlockingQueue<>(1000);
- this.isComplete = new AtomicBoolean(false);
- this.previousPullWasEmpty = false;
- }
-
- @Override
- public void startStream() {
- BackOffStrategy backOffStrategy = new ExponentialBackOffStrategy(2);
-
- for (UserInfo user : this.config.getYoutubeUsers()) {
- if (this.config.getDefaultAfterDate() != null && user.getAfterDate() == null) {
- user.setAfterDate(this.config.getDefaultAfterDate());
- }
- if (this.config.getDefaultBeforeDate() != null && user.getBeforeDate() == null) {
- user.setBeforeDate(this.config.getDefaultBeforeDate());
- }
-
- ListenableFuture future = executor.submit(getDataCollector(backOffStrategy, this.datumQueue, this.youtube, user));
- futures.add(future);
- }
-
- this.executor.shutdown();
- }
-
- protected abstract Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo);
-
- @Override
- public StreamsResultSet readCurrent() {
- BlockingQueue<StreamsDatum> batch = new LinkedBlockingQueue<>();
- int batchCount = 0;
- while (!this.datumQueue.isEmpty() && batchCount < MAX_BATCH_SIZE) {
- StreamsDatum datum = ComponentUtils.pollWhileNotEmpty(this.datumQueue);
- if (datum != null) {
- ++batchCount;
- ComponentUtils.offerUntilSuccess(datum, batch);
- }
- }
- return new StreamsResultSet(batch);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @VisibleForTesting
- protected YouTube createYouTubeClient() throws IOException, GeneralSecurityException {
- GoogleCredential.Builder credentialBuilder = new GoogleCredential.Builder()
- .setTransport(HTTP_TRANSPORT)
- .setJsonFactory(JSON_FACTORY)
- .setServiceAccountId(getConfig().getOauth().getServiceAccountEmailAddress())
- .setServiceAccountScopes(scopes);
-
- if (StringUtils.isNotEmpty(getConfig().getOauth().getPathToP12KeyFile())) {
- File p12KeyFile = new File(getConfig().getOauth().getPathToP12KeyFile());
- if (p12KeyFile.exists() && p12KeyFile.isFile() && p12KeyFile.canRead()) {
- credentialBuilder = credentialBuilder.setServiceAccountPrivateKeyFromP12File(p12KeyFile);
- }
- }
- Credential credential = credentialBuilder.build();
- return new YouTube.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential).setApplicationName("Streams Application").build();
- }
-
- @Override
- public void cleanUp() {
- ComponentUtils.shutdownExecutor(this.executor, 10, 10);
- this.executor = null;
- }
-
- public YoutubeConfiguration getConfig() {
- return config;
- }
-
- public void setConfig(YoutubeConfiguration config) {
- this.config = config;
- }
-
- /**
- * Set and overwrite the default before date that was read from the configuration file.
- * @param defaultBeforeDate defaultBeforeDate
- */
- public void setDefaultBeforeDate(DateTime defaultBeforeDate) {
- this.config.setDefaultBeforeDate(defaultBeforeDate);
- }
-
- /**
- * Set and overwrite the default after date that was read from teh configuration file.
- * @param defaultAfterDate defaultAfterDate
- */
- public void setDefaultAfterDate(DateTime defaultAfterDate) {
- this.config.setDefaultAfterDate(defaultAfterDate);
- }
-
- /**
- * Sets and overwrite the user info from the configuaration file. Uses the defaults before and after dates.
- * @param userIds Set of String userIds
- */
- public void setUserInfoWithDefaultDates(Set<String> userIds) {
- List<UserInfo> youtubeUsers = new LinkedList<>();
-
- for (String userId : userIds) {
- UserInfo user = new UserInfo();
- user.setUserId(userId);
- user.setAfterDate(this.config.getDefaultAfterDate());
- user.setBeforeDate(this.config.getDefaultBeforeDate());
- youtubeUsers.add(user);
- }
-
- this.config.setYoutubeUsers(youtubeUsers);
- }
-
- /**
- * Set and overwrite user into from teh configuration file. Only sets after dater.
- * @param usersAndAfterDates usersAndAfterDates
- */
- public void setUserInfoWithAfterDate(Map<String, DateTime> usersAndAfterDates) {
- List<UserInfo> youtubeUsers = new LinkedList<>();
-
- for (String userId : usersAndAfterDates.keySet()) {
- UserInfo user = new UserInfo();
- user.setUserId(userId);
- user.setAfterDate(usersAndAfterDates.get(userId));
- youtubeUsers.add(user);
- }
-
- this.config.setYoutubeUsers(youtubeUsers);
- }
-
- @Override
- public boolean isRunning() {
- if (datumQueue.isEmpty() && executor.isTerminated() && Futures.allAsList(futures).isDone()) {
- LOGGER.info("Completed");
- isComplete.set(true);
- LOGGER.info("Exiting");
- }
- return !isComplete.get();
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollector.java
deleted file mode 100644
index ad94146..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollector.java
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.youtube.YoutubeConfiguration;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.services.youtube.YouTube;
-import com.google.api.services.youtube.model.ActivityListResponse;
-import com.google.api.services.youtube.model.Video;
-import com.google.api.services.youtube.model.VideoListResponse;
-import com.google.gson.Gson;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-/**
- * YoutubeDataCollector for YoutubeUserActivityProvider.
- */
-public class YoutubeUserActivityCollector extends YoutubeDataCollector {
-
- /**
- * Max results allowed per request
- * https://developers.google.com/+/api/latest/activities/list
- */
- private static final long MAX_RESULTS = 50;
- private static final int MAX_ATTEMPTS = 5;
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeUserActivityCollector.class);
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- static { //set up mapper for Google Activity Object
- SimpleModule simpleModule = new SimpleModule();
- MAPPER.registerModule(simpleModule);
- MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- private Gson gson = new Gson();
- private BlockingQueue<StreamsDatum> datumQueue;
- private BackOffStrategy backOff;
- private YouTube youtube;
- private UserInfo userInfo;
- private YoutubeConfiguration config;
-
- /**
- * YoutubeUserActivityCollector constructor.
- * @param youtube YouTube
- * @param datumQueue BlockingQueue of StreamsDatum
- * @param backOff BackOffStrategy
- * @param userInfo UserInfo
- * @param config YoutubeConfiguration
- */
- public YoutubeUserActivityCollector(
- YouTube youtube,
- BlockingQueue<StreamsDatum> datumQueue,
- BackOffStrategy backOff,
- UserInfo userInfo,
- YoutubeConfiguration config) {
- this.youtube = youtube;
- this.datumQueue = datumQueue;
- this.backOff = backOff;
- this.userInfo = userInfo;
- this.config = config;
- }
-
- @Override
- public void run() {
- collectActivityData();
- }
-
- /**
- * Iterate through all users in the Youtube configuration and collect all videos
- * associated with their accounts.
- */
- protected void collectActivityData() {
-
- int item_count = 0;
- int last_count = 0;
- int page_count = 0;
-
- try {
- YouTube.Activities.List request = null;
- ActivityListResponse feed = null;
-
- boolean tryAgain = false;
- int attempt = 0;
- DateTime afterDate = userInfo.getAfterDate();
- DateTime beforeDate = userInfo.getBeforeDate();
-
- do {
- try {
- if (request == null) {
- request = this.youtube.activities().list("contentDetails")
- .setChannelId(userInfo.getUserId())
- .setMaxResults(MAX_RESULTS)
- .setKey(config.getApiKey());
- feed = request.execute();
- } else {
- request = this.youtube.activities().list("contentDetails")
- .setChannelId(userInfo.getUserId())
- .setMaxResults(MAX_RESULTS)
- .setPageToken(feed.getNextPageToken())
- .setKey(config.getApiKey());
- feed = request.execute();
- }
- this.backOff.reset(); //successful pull reset api.
- last_count = feed.getItems().size();
- processActivityFeed(feed, afterDate, beforeDate);
- item_count += last_count;
- page_count++;
- } catch (GoogleJsonResponseException gjre) {
- tryAgain = backoffAndIdentifyIfRetry(gjre, this.backOff);
- ++attempt;
- }
- }
- while ((tryAgain || (feed != null && feed.getNextPageToken() != null)) && attempt < MAX_ATTEMPTS);
- } catch (Throwable throwable) {
- if (throwable instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- throwable.printStackTrace();
- LOGGER.warn("Unable to pull Activities for user={} : {}", this.userInfo.getUserId(), throwable);
- }
-
- LOGGER.info("item_count: {} last_count: {} page_count: {} ", item_count, last_count, page_count);
-
- }
-
- /**
- * Given a feed and an after and before date, fetch all relevant user videos
- * and place them into the datumQueue for post-processing.
- * @param feed ActivityListResponse
- * @param afterDate DateTime
- * @param beforeDate DateTime
- * @throws IOException IOException
- * @throws InterruptedException InterruptedException
- */
- void processActivityFeed(ActivityListResponse feed, DateTime afterDate, DateTime beforeDate) throws IOException, InterruptedException {
- for (com.google.api.services.youtube.model.Activity activity : feed.getItems()) {
- try {
- List<Video> videos = new ArrayList<>();
-
- if (activity.getContentDetails().getUpload() != null) {
- videos.addAll(getVideoList(activity.getContentDetails().getUpload().getVideoId()));
- }
- if (activity.getContentDetails().getPlaylistItem() != null && activity.getContentDetails().getPlaylistItem().getResourceId() != null) {
- videos.addAll(getVideoList(activity.getContentDetails().getPlaylistItem().getResourceId().getVideoId()));
- }
-
- processVideos(videos, afterDate, beforeDate, activity, feed);
- } catch (Exception ex) {
- LOGGER.error("Error while trying to process activity: {}, {}", activity, ex);
- }
- }
- }
-
- /**
- * Process a list of Video objects.
- * @param videos List of Video
- * @param afterDate afterDate
- * @param beforeDate beforeDate
- * @param activity com.google.api.services.youtube.model.Activity
- * @param feed ActivityListResponse
- */
- void processVideos(List<Video> videos, DateTime afterDate, DateTime beforeDate, com.google.api.services.youtube.model.Activity activity, ActivityListResponse feed) {
- try {
- for (Video video : videos) {
- if (video != null) {
- org.joda.time.DateTime published = new org.joda.time.DateTime(video.getSnippet().getPublishedAt().getValue());
- if ((afterDate == null && beforeDate == null)
- || (beforeDate == null && afterDate.isBefore(published))
- || (afterDate == null && beforeDate.isAfter(published))
- || ((afterDate != null && beforeDate != null) && (afterDate.isAfter(published) && beforeDate.isBefore(published)))) {
- LOGGER.debug("Providing Youtube Activity: {}", MAPPER.writeValueAsString(video));
- this.datumQueue.put(new StreamsDatum(gson.toJson(video), activity.getId()));
- } else if (afterDate != null && afterDate.isAfter(published)) {
- feed.setNextPageToken(null); // do not fetch next page
- break;
- }
- }
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to process video list: {}, {}", videos, ex);
- }
- }
-
- /**
- * Given a Youtube videoId, return the relevant Youtube Video object.
- * @param videoId videoId
- * @return List of Videos
- * @throws IOException
- */
- List<Video> getVideoList(String videoId) throws IOException {
- VideoListResponse videosListResponse = this.youtube.videos().list("snippet,statistics")
- .setId(videoId)
- .setKey(config.getApiKey())
- .execute();
-
- if (videosListResponse.getItems().size() == 0) {
- LOGGER.debug("No Youtube videos found for videoId: {}", videoId);
- return new ArrayList<>();
- }
-
- return videosListResponse.getItems();
- }
-
- BlockingQueue<StreamsDatum> getDatumQueue() {
- return this.datumQueue;
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityProvider.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityProvider.java
deleted file mode 100644
index d4719ce..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/provider/YoutubeUserActivityProvider.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.provider;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.youtube.YoutubeConfiguration;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.api.services.youtube.YouTube;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigParseOptions;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.util.Iterator;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Retrieve recent activity from a list of user ids or names.
- */
-public class YoutubeUserActivityProvider extends YoutubeProvider {
-
- public YoutubeUserActivityProvider() {
- super();
- }
-
- public YoutubeUserActivityProvider(YoutubeConfiguration config) {
- super(config);
- }
-
- /**
- * To use from command line:
- * <p>
- * <p/>
- * Supply (at least) the following required configuration in application.conf:
- * <p>
- * <p/>
- * youtube.oauth.pathToP12KeyFile
- * youtube.oauth.serviceAccountEmailAddress
- * youtube.apiKey
- * youtube.youtubeUsers
- * <p>
- * <p/>
- * Launch using:
- * <p>
- * <p/>
- * mvn exec:java -Dexec.mainClass=org.apache.streams.youtube.provider.YoutubeUserActivityProvider -Dexec.args="application.conf tweets.json"
- *
- * @param args args
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- Preconditions.checkArgument(args.length >= 2);
-
- String configfile = args[0];
- String outfile = args[1];
-
- Config reference = ConfigFactory.load();
- File file = new File(configfile);
- assert (file.exists());
- Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false));
-
- Config typesafe = testResourceConfig.withFallback(reference).resolve();
-
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe);
- YoutubeConfiguration config = new ComponentConfigurator<>(YoutubeConfiguration.class).detectConfiguration(typesafe, "youtube");
- YoutubeUserActivityProvider provider = new YoutubeUserActivityProvider(config);
-
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile)));
- provider.prepare(config);
- provider.startStream();
- do {
- Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- Iterator<StreamsDatum> iterator = provider.readCurrent().iterator();
- while (iterator.hasNext()) {
- StreamsDatum datum = iterator.next();
- String json;
- try {
- if (datum.getDocument() instanceof String) {
- json = (String) datum.getDocument();
- } else {
- json = mapper.writeValueAsString(datum.getDocument());
- }
- outStream.println(json);
- } catch (JsonProcessingException ex) {
- System.err.println(ex.getMessage());
- }
- }
- }
- while (provider.isRunning());
- provider.cleanUp();
- outStream.flush();
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo) {
- return new YoutubeUserActivityCollector(youtube, queue, strategy, userInfo, config);
- }
-}
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeActivityUtil.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeActivityUtil.java
deleted file mode 100644
index 4b985f6..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeActivityUtil.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.serializer;
-
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Image;
-import org.apache.streams.pojo.json.Provider;
-
-import com.google.api.services.youtube.YouTube;
-import com.google.api.services.youtube.model.Channel;
-import com.google.api.services.youtube.model.Thumbnail;
-import com.google.api.services.youtube.model.ThumbnailDetails;
-import com.google.api.services.youtube.model.Video;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public class YoutubeActivityUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeActivityUtil.class);
-
- /**
- * Given a {@link YouTube.Videos} object and an
- * {@link Activity} object, fill out the appropriate details
- *
- * @param video Video
- * @param activity Activity
- * @throws ActivitySerializerException ActivitySerializerException
- */
- public static void updateActivity(Video video, Activity activity, String channelId) throws ActivitySerializerException {
- activity.setActor(buildActor(video, video.getSnippet().getChannelId()));
- activity.setVerb("post");
-
- activity.setId(formatId(activity.getVerb(), Optional.ofNullable(video.getId()).orElse(null)));
-
- activity.setPublished(new DateTime(video.getSnippet().getPublishedAt().getValue()));
- activity.setTitle(video.getSnippet().getTitle());
- activity.setContent(video.getSnippet().getDescription());
- activity.setUrl("https://www.youtube.com/watch?v=" + video.getId());
-
- activity.setProvider(getProvider());
-
- activity.setObject(buildActivityObject(video));
-
- addYoutubeExtensions(activity, video);
- }
-
-
- /**
- * Given a {@link Channel} object and an
- * {@link Activity} object, fill out the appropriate details
- *
- * @param channel Channel
- * @param activity Activity
- * @throws ActivitySerializerException ActivitySerializerException
- */
- public static void updateActivity(Channel channel, Activity activity, String channelId) throws ActivitySerializerException {
- try {
- activity.setProvider(getProvider());
- activity.setVerb("post");
- activity.setActor(createActorForChannel(channel));
- Map<String, Object> extensions = new HashMap<>();
- extensions.put("youtube", channel);
- activity.setAdditionalProperty("extensions", extensions);
- } catch (Throwable throwable) {
- throw new ActivitySerializerException(throwable);
- }
- }
-
- /**
- * createActorForChannel.
- * @param channel Channel
- * @return $.actor
- */
- public static ActivityObject createActorForChannel(Channel channel) {
- ActivityObject actor = new ActivityObject();
- // TODO: use generic provider id concatenator
- actor.setId("id:youtube:" + channel.getId());
- actor.setSummary(channel.getSnippet().getDescription());
- actor.setDisplayName(channel.getSnippet().getTitle());
- Image image = new Image();
- image.setUrl(channel.getSnippet().getThumbnails().getHigh().getUrl());
- actor.setImage(image);
- actor.setUrl("https://youtube.com/user/" + channel.getId());
- Map<String, Object> actorExtensions = new HashMap<>();
- actorExtensions.put("followers", channel.getStatistics().getSubscriberCount());
- actorExtensions.put("posts", channel.getStatistics().getVideoCount());
- actor.setAdditionalProperty("extensions", actorExtensions);
- return actor;
- }
-
- /**
- * Given a video object, create the appropriate activity object with a valid image
- * (thumbnail) and video URL.
- * @param video Video
- * @return Activity Object with Video URL and a thumbnail image
- */
- private static ActivityObject buildActivityObject(Video video) {
- ActivityObject activityObject = new ActivityObject();
-
- ThumbnailDetails thumbnailDetails = video.getSnippet().getThumbnails();
- Thumbnail thumbnail = thumbnailDetails.getDefault();
-
- if (thumbnail != null) {
- Image image = new Image();
- image.setUrl(thumbnail.getUrl());
- image.setHeight(thumbnail.getHeight());
- image.setWidth(thumbnail.getWidth());
-
- activityObject.setImage(image);
- }
-
- activityObject.setUrl("https://www.youtube.com/watch?v=" + video.getId());
- activityObject.setObjectType("video");
-
- return activityObject;
- }
-
- /**
- * Add the Youtube extensions to the Activity object that we're building.
- * @param activity Activity
- * @param video Video
- */
- private static void addYoutubeExtensions(Activity activity, Video video) {
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
-
- extensions.put("youtube", video);
-
- if (video.getStatistics() != null) {
- Map<String, Object> likes = new HashMap<>();
- likes.put("count", video.getStatistics().getCommentCount());
- extensions.put("likes", likes);
- }
- }
-
- /**
- * Build an {@link ActivityObject} actor given the video object
- * @param video Video
- * @param id id
- * @return Actor object
- */
- private static ActivityObject buildActor(Video video, String id) {
- ActivityObject actor = new ActivityObject();
-
- actor.setId("id:youtube:" + id);
- actor.setDisplayName(video.getSnippet().getChannelTitle());
- actor.setSummary(video.getSnippet().getDescription());
- actor.setAdditionalProperty("handle", video.getSnippet().getChannelTitle());
-
- return actor;
- }
-
- /**
- * Gets the common youtube {@link Provider} object
- * @return a provider object representing YouTube
- */
- public static Provider getProvider() {
- Provider provider = new Provider();
- provider.setId("id:providers:youtube");
- provider.setDisplayName("YouTube");
- return provider;
- }
-
- /**
- * Formats the ID to conform with the Apache Streams activity ID convention
- * @param idparts the parts of the ID to join
- * @return a valid Activity ID in format "id:youtube:part1:part2:...partN"
- */
- public static String formatId(String... idparts) {
- return String.join(":",
- Stream.concat(Arrays.stream(new String[] {"id:youtube"}), Arrays.stream(idparts)).collect(Collectors.toList()));
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeChannelDeserializer.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeChannelDeserializer.java
deleted file mode 100644
index e5d0ed0..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeChannelDeserializer.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.serializer;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.api.client.util.DateTime;
-import com.google.api.services.youtube.model.Channel;
-import com.google.api.services.youtube.model.ChannelContentDetails;
-import com.google.api.services.youtube.model.ChannelLocalization;
-import com.google.api.services.youtube.model.ChannelSnippet;
-import com.google.api.services.youtube.model.ChannelStatistics;
-import com.google.api.services.youtube.model.ChannelTopicDetails;
-import com.google.api.services.youtube.model.Thumbnail;
-import com.google.api.services.youtube.model.ThumbnailDetails;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * YoutubeChannelDeserializer is a JsonDeserializer for Channel.
- */
-public class YoutubeChannelDeserializer extends JsonDeserializer<Channel> {
-
- @Override
- public Channel deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException {
- JsonNode node = jp.getCodec().readTree(jp);
- try {
- Channel channel = new Channel();
- if (node.findPath("etag") != null) {
- channel.setEtag(node.get("etag").asText());
- }
- if (node.findPath("kind") != null) {
- channel.setKind(node.get("kind").asText());
- }
- channel.setId(node.get("id").asText());
- channel.setTopicDetails(setTopicDetails(node.findValue("topicDetails")));
- channel.setStatistics(setChannelStatistics(node.findValue("statistics")));
- channel.setContentDetails(setContentDetails(node.findValue("contentDetails")));
- channel.setSnippet(setChannelSnippet(node.findValue("snippet")));
- return channel;
- } catch (Throwable throwable) {
- throw new IOException(throwable);
- }
- }
-
- protected ChannelSnippet setChannelSnippet(JsonNode node) {
- ChannelSnippet snippet = new ChannelSnippet();
- snippet.setTitle(node.get("title").asText());
- snippet.setDescription(node.get("description").asText());
- snippet.setPublishedAt(new DateTime(node.get("publishedAt").get("value").longValue()));
- snippet.setLocalized(setLocalized(node.findValue("localized")));
- snippet.setThumbnails(setThumbnails(node.findValue("thumbnails")));
- return snippet;
- }
-
- protected ThumbnailDetails setThumbnails(JsonNode node) {
- ThumbnailDetails details = new ThumbnailDetails();
- if (node == null) {
- return details;
- }
- details.setDefault(new Thumbnail().setUrl(node.get("default").get("url").asText()));
- details.setHigh(new Thumbnail().setUrl(node.get("high").get("url").asText()));
- details.setMedium(new Thumbnail().setUrl(node.get("medium").get("url").asText()));
- return details;
- }
-
- protected ChannelLocalization setLocalized(JsonNode node) {
- if (node == null) {
- return new ChannelLocalization();
- }
- ChannelLocalization localization = new ChannelLocalization();
- localization.setDescription(node.get("description").asText());
- localization.setTitle(node.get("title").asText());
- return localization;
- }
-
- protected ChannelContentDetails setContentDetails(JsonNode node) {
- ChannelContentDetails contentDetails = new ChannelContentDetails();
- if (node == null) {
- return contentDetails;
- }
- if (node.findValue("googlePlusUserId") != null) {
- contentDetails.setGooglePlusUserId(node.get("googlePlusUserId").asText());
- }
- contentDetails.setRelatedPlaylists(setRelatedPlaylists(node.findValue("relatedPlaylists")));
- return contentDetails;
- }
-
- protected ChannelContentDetails.RelatedPlaylists setRelatedPlaylists(JsonNode node) {
- ChannelContentDetails.RelatedPlaylists playlists = new ChannelContentDetails.RelatedPlaylists();
- if (node == null) {
- return playlists;
- }
- if (node.findValue("favorites") != null) {
- playlists.setFavorites(node.get("favorites").asText());
- }
- if (node.findValue("likes") != null) {
- playlists.setLikes(node.get("likes").asText());
- }
- if (node.findValue("uploads") != null) {
- playlists.setUploads(node.get("uploads").asText());
- }
- return playlists;
- }
-
- protected ChannelStatistics setChannelStatistics(JsonNode node) {
- ChannelStatistics stats = new ChannelStatistics();
- if (node == null) {
- return stats;
- }
- stats.setCommentCount(node.get("commentCount").bigIntegerValue());
- stats.setHiddenSubscriberCount(node.get("hiddenSubscriberCount").asBoolean());
- stats.setSubscriberCount(node.get("subscriberCount").bigIntegerValue());
- stats.setVideoCount(node.get("videoCount").bigIntegerValue());
- stats.setViewCount(node.get("viewCount").bigIntegerValue());
- return stats;
- }
-
- protected ChannelTopicDetails setTopicDetails(JsonNode node) {
- ChannelTopicDetails details = new ChannelTopicDetails();
- if (node == null) {
- return details;
- }
- List<String> topicIds = new LinkedList<>();
- for (JsonNode jsonNode : node.get("topicIds")) {
- topicIds.add(jsonNode.asText());
- }
- details.setTopicIds(topicIds);
- return details;
- }
-}
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeEventClassifier.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeEventClassifier.java
deleted file mode 100644
index 964d6ca..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeEventClassifier.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.serializer;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.api.services.youtube.model.Video;
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.IOException;
-import java.util.Objects;
-
-public class YoutubeEventClassifier {
- private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- private static final String VIDEO_IDENTIFIER = "\"youtube#video\"";
- private static final String CHANNEL_IDENTIFIER = "youtube#channel";
-
- /**
- * detect probable Class of a json String from YouTube.
- * @param json json
- * @return Class
- */
- public static Class detectClass(String json) {
- Objects.requireNonNull(json);
- Preconditions.checkArgument(StringUtils.isNotEmpty(json));
-
- ObjectNode objectNode;
- try {
- objectNode = (ObjectNode) mapper.readTree(json);
- } catch (IOException ex) {
- ex.printStackTrace();
- return null;
- }
-
- if (objectNode.findValue("kind") != null && objectNode.get("kind").toString().equals(VIDEO_IDENTIFIER)) {
- return Video.class;
- } else if (objectNode.findValue("kind") != null && objectNode.get("kind").toString().contains(CHANNEL_IDENTIFIER)) {
- return com.google.api.services.youtube.model.Channel.class;
- } else {
- return ObjectNode.class;
- }
- }
-}
diff --git a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeVideoDeserializer.java b/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeVideoDeserializer.java
deleted file mode 100644
index 24f8e51..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/java/org/apache/streams/youtube/serializer/YoutubeVideoDeserializer.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.serializer;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.api.client.util.DateTime;
-import com.google.api.services.youtube.model.Thumbnail;
-import com.google.api.services.youtube.model.ThumbnailDetails;
-import com.google.api.services.youtube.model.Video;
-import com.google.api.services.youtube.model.VideoSnippet;
-import com.google.api.services.youtube.model.VideoStatistics;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-public class YoutubeVideoDeserializer extends JsonDeserializer<Video> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeVideoDeserializer.class);
-
- /**
- * Because the Youtube Video object contains complex objects within its hierarchy, we have to use
- * a custom deserializer
- *
- * @param jsonParser jsonParser
- * @param deserializationContext deserializationContext
- * @return The deserialized {@link com.google.api.services.youtube.YouTube.Videos} object
- * @throws java.io.IOException IOException
- * @throws com.fasterxml.jackson.core.JsonProcessingException JsonProcessingException
- */
- @Override
- public Video deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
- JsonNode node = jsonParser.getCodec().readTree(jsonParser);
- Video video = new Video();
-
- try {
- video.setId(node.get("id").asText());
- video.setEtag(node.get("etag").asText());
- video.setKind(node.get("kind").asText());
-
- video.setSnippet(buildSnippet(node));
- video.setStatistics(buildStatistics(node));
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserialize a Video object: {}", ex);
- }
-
- return video;
- }
-
- /**
- * Given the raw JsonNode, construct a video snippet object.
- * @param node JsonNode
- * @return VideoSnippet
- */
- private VideoSnippet buildSnippet(JsonNode node) {
- VideoSnippet snippet = new VideoSnippet();
- JsonNode snippetNode = node.get("snippet");
-
- snippet.setChannelId(snippetNode.get("channelId").asText());
- snippet.setChannelTitle(snippetNode.get("channelTitle").asText());
- snippet.setDescription(snippetNode.get("description").asText());
- snippet.setTitle(snippetNode.get("title").asText());
- snippet.setPublishedAt(new DateTime(snippetNode.get("publishedAt").get("value").asLong()));
-
- ThumbnailDetails thumbnailDetails = new ThumbnailDetails();
- for (JsonNode t : snippetNode.get("thumbnails")) {
- Thumbnail thumbnail = new Thumbnail();
-
- thumbnail.setHeight(t.get("height").asLong());
- thumbnail.setUrl(t.get("url").asText());
- thumbnail.setWidth(t.get("width").asLong());
-
- thumbnailDetails.setDefault(thumbnail);
- }
-
- snippet.setThumbnails(thumbnailDetails);
-
- return snippet;
- }
-
- /**
- * Given the raw JsonNode, construct a statistics object.
- * @param node JsonNode
- * @return VideoStatistics
- */
- private VideoStatistics buildStatistics(JsonNode node) {
- VideoStatistics statistics = new VideoStatistics();
- JsonNode statisticsNode = node.get("statistics");
-
- statistics.setCommentCount(statisticsNode.get("commentCount").bigIntegerValue());
- statistics.setDislikeCount(statisticsNode.get("dislikeCount").bigIntegerValue());
- statistics.setFavoriteCount(statisticsNode.get("favoriteCount").bigIntegerValue());
- statistics.setLikeCount(statisticsNode.get("likeCount").bigIntegerValue());
- statistics.setViewCount(statisticsNode.get("viewCount").bigIntegerValue());
-
- return statistics;
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-youtube/src/main/jsonschema/org/apache/streams/youtube/YoutubeConfiguration.json b/streams-contrib/streams-provider-youtube/src/main/jsonschema/org/apache/streams/youtube/YoutubeConfiguration.json
deleted file mode 100644
index 7e80731..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/jsonschema/org/apache/streams/youtube/YoutubeConfiguration.json
+++ /dev/null
@@ -1,105 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.youtube.YoutubeConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "protocol": {
- "type": "string",
- "description": "The protocol"
- },
- "host": {
- "type": "string",
- "description": "The host"
- },
- "port": {
- "type": "integer",
- "description": "The port"
- },
- "version": {
- "type": "string",
- "description": "The version"
- },
- "endpoint": {
- "type": "string",
- "description": "The endpoint"
- },
- "apiKey": {
- "type": "string",
- "description": "API key to allow for authenticated (but not owned) requests"
- },
- "follow": {
- "type": "array",
- "description": "DEPRECATED. A list of user names, indicating the users whose activities should be delivered on the stream",
- "items": {
- "type": "string"
- }
- },
- "youtubeUsers": {
- "type": "array",
- "description": "A list of user user ids and optional date parameters for the GPlus provider",
- "items": {
- "type": "object",
- "$ref": "#/definitions/userInfo"
- }
- },
- "defaultAfterDate": {
- "type": "string",
- "format": "date-time",
- "description": "Optional parameter for the provider. If this value is not null an the afterDate value in the userInfo is null, this value will be used."
- },
- "defaultBeforeDate": {
- "type": "string",
- "format": "date-time",
- "description": "Optional parameter for the provider. If this value is not null and the beforeDate value in the userInfo is null, this value will be used."
- },
- "oauth": {
- "type": "object",
- "dynamic": "true",
- "javaType" : "org.apache.streams.google.gplus.GPlusOAuthConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "DEPRICATED",
- "properties": {
- "appName": {
- "type": "string"
- },
- "pathToP12KeyFile": {
- "type": "string",
- "description": "Absolute Path to key file"
- },
- "serviceAccountEmailAddress": {
- "type": "string",
- "description": "Service Account email address for your app"
- }
- }
- }
- },
- "definitions": {
- "userInfo": {
- "type": "object",
- "javaInterfaces" : ["java.io.Serializable"],
- "dynamic": "true",
- "javaType": "org.apache.streams.google.gplus.configuration.UserInfo",
- "properties": {
- "userId": {
- "type": "string",
- "description": "Google+ user id"
- },
- "afterDate": {
- "type": "string",
- "format": "date-time",
- "description": "If the api allows to gather data by date range, this date will be used as the start of the range for the request for this user. If this is null it will use the defaultBeforeDate."
- },
- "beforeDate": {
- "type": "string",
- "format": "date-time",
- "description": "If the api allows to gather data by date range, this date will be used as the end of the range for the request for this user.. If this is null it will use the defaultAfterDate."
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-youtube/src/main/resources/components.dot b/streams-contrib/streams-provider-youtube/src/main/resources/components.dot
deleted file mode 100644
index 4d3eb22..0000000
--- a/streams-contrib/streams-provider-youtube/src/main/resources/components.dot
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-digraph g {
-
- graph [compound = true];
-
- //presentation
- splines = true;
- overlap = false;
- rankdir = TB;
-
- subgraph cluster_generators {
- label="generators";
- style="dotted";
- generators_youtube [label="https://api.youtube.com"]
- }
-
- subgraph cluster_providers {
- label="providers";
- providers_youtube_channel [label="YoutubeChannelProvider"]
- providers_youtube_video [label="YoutubeUserActivityProvider"]
- }
-
- subgraph cluster_processors {
- label="processors";
- processors_youtube_activity [label="YoutubeTypeConverter"]
- processors_youtube_activityobject [label="YoutubeTypeConverter"]
- }
-
- persisters [label="persisters", shape="cylinder"];
-
- generators_youtube -> providers_youtube_channel
- generators_youtube -> providers_youtube_video
- providers_youtube_channel -> processors_youtube_activityobject [label="o.a.s.y.Channel"]
- providers_youtube_video -> processors_youtube_activity [label="o.a.s.y.Video"]
-
- processors_youtube_activityobject -> persisters [label="o.a.s.p.j.Page"]
- processors_youtube_activity -> persisters [label="o.a.s.p.j.Video\no.a.s.p.j.Comment"]
-
-}
diff --git a/streams-contrib/streams-provider-youtube/src/site/markdown/index.md b/streams-contrib/streams-provider-youtube/src/site/markdown/index.md
deleted file mode 100644
index de050e9..0000000
--- a/streams-contrib/streams-provider-youtube/src/site/markdown/index.md
+++ /dev/null
@@ -1,50 +0,0 @@
-org.apache.streams:streams-provider-youtube
-===========================================
-
-streams-provider-youtube contains providers, conversions, and utility classes.
-
-## Data Types
-
-This module relies on classes from com.google.apis:google-api-services-youtube
-
-## Configuration
-
-| Schema |
-|--------|
-| [YoutubeConfiguration.json](com/youtube/YoutubeConfiguration.json "YoutubeConfiguration.json") [YoutubeConfiguration.html](apidocs/com/youtube/Youtube.html "javadoc") |
-
-## Components
-
-
-
-| Class | Configuration |
-|-------|---------------|
-| YoutubeChannelProvider [YoutubeChannelProvider.html](apidocs/com/youtube/provider/YoutubeChannelProvider.html "javadoc") | [YoutubeConfiguration.json](com/youtube/YoutubeConfiguration.json "YoutubeConfiguration.json") [YoutubeConfiguration.html](apidocs/com/youtube/YoutubeConfiguration.html "javadoc")
-| YoutubeUserActivityProvider [YoutubeUserActivityProvider.html](apidocs/com/youtube/provider/YoutubeUserActivityProvider.html "javadoc") | [YoutubeConfiguration.json](com/youtube/YoutubeConfiguration.json "YoutubeConfiguration.json") [YoutubeConfiguration.html](apidocs/com/youtube/YoutubeConfiguration.html "javadoc")
-
-Test:
------
-
-Log into admin console
-Create project
-Enable Data API on project
-Create service account
-Download p12 file
-
-Create a local file `youtube.conf` with valid youtube credentials
-
- youtube {
- apiKey = ""
- oauth {
- serviceAccountEmailAddress = ""
- pathToP12KeyFile = ""
- }
- }
-
-Build with integration testing enabled, using your credentials
-
- mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/youtube.conf"
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/processor/YoutubeTypeConverterTest.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/processor/YoutubeTypeConverterTest.java
deleted file mode 100644
index fc5a309..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/processor/YoutubeTypeConverterTest.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.processor;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.youtube.serializer.YoutubeActivityUtil;
-import org.apache.streams.youtube.serializer.YoutubeVideoDeserializer;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.youtube.model.Video;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * Test for YoutubeTypeConverter.
- */
-public class YoutubeTypeConverterTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeTypeConverterTest.class);
- private final String testVideo = "{\"etag\":\"\\\"4FSIjSQU83ZJMYAO0IqRYMvZX98/V0q3OIauZ3ZAkszLUDbHL45yEGM\\\"\",\"id\":\"sUOepRctwVE\",\"kind\":\"youtube#video\",\"snippet\":{\"channelId\":\"UCNENOn2nmwguQYkejKhJGPQ\",\"channelTitle\":\"Carilion Clinic\",\"description\":\"Join Carilion Clinic's Heart Failure experts for a LIVE Google+ Hangout on Feb. 23, 12:30-1 p.m. to learn more about heart failure, treatment options, and lifestyle changes. Learn more: https://plus.google.com/u/0/eve [...]
-
- private YoutubeTypeConverter youtubeTypeConverter;
- private ObjectMapper objectMapper;
-
- /**
- * setup for test.
- */
- @Before
- public void setup() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Video.class, new YoutubeVideoDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
- youtubeTypeConverter = new YoutubeTypeConverter();
- youtubeTypeConverter.prepare(null);
- }
-
- @Test
- public void testVideoConversion() {
- try {
- LOGGER.info("raw: {}", testVideo);
- Activity activity = new Activity();
-
- Video video = objectMapper.readValue(testVideo, Video.class);
- StreamsDatum streamsDatum = new StreamsDatum(video);
-
- assertNotNull(streamsDatum.getDocument());
-
- List<StreamsDatum> retList = youtubeTypeConverter.process(streamsDatum);
- YoutubeActivityUtil.updateActivity(video, activity, "testChannelId");
-
- assertEquals(retList.size(), 1);
- assert (retList.get(0).getDocument() instanceof Activity);
- assertEquals(activity, retList.get(0).getDocument());
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to convert video to activity: {}", ex);
- }
- }
-
- @Test
- public void testStringVideoConversion() {
- try {
- LOGGER.info("raw: {}", testVideo);
- Activity activity = new Activity();
-
- Video video = objectMapper.readValue(testVideo, Video.class);
- StreamsDatum streamsDatum = new StreamsDatum(testVideo);
-
- assertNotNull(streamsDatum.getDocument());
-
- List<StreamsDatum> retList = youtubeTypeConverter.process(streamsDatum);
- YoutubeActivityUtil.updateActivity(video, activity, "testChannelId");
-
- assertEquals(retList.size(), 1);
- assert (retList.get(0).getDocument() instanceof Activity);
- assertEquals(activity, retList.get(0).getDocument());
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to convert video to activity: {}", ex);
- }
- }
-
-}
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollectorTest.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollectorTest.java
deleted file mode 100644
index b291760..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeChannelDataCollectorTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.util.api.requests.backoff.impl.LinearTimeBackOffStrategy;
-import org.apache.streams.youtube.YoutubeConfiguration;
-
-import com.google.api.services.youtube.YouTube;
-import com.google.api.services.youtube.model.Channel;
-import com.google.api.services.youtube.model.ChannelListResponse;
-import org.junit.Test;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * YoutubeChannelDataCollectorTest tests YoutubeChannelDataCollector.
- */
-public class YoutubeChannelDataCollectorTest {
-
- private static final String ID = "12345";
-
- @Test
- public void testDataCollector() throws Exception {
- YouTube youTube = createMockYoutube();
- BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
- BackOffStrategy strategy = new LinearTimeBackOffStrategy(1);
- UserInfo userInfo = new UserInfo();
- userInfo.setUserId(ID);
- YoutubeConfiguration config = new YoutubeConfiguration();
- config.setApiKey(ID);
- YoutubeChannelDataCollector collector = new YoutubeChannelDataCollector(youTube, queue, strategy, userInfo, config);
- collector.run();
- assertEquals(1, queue.size());
- StreamsDatum datum = queue.take();
- assertNotNull(datum);
- String document = (String) datum.getDocument();
- assertNotNull(document);
- }
-
- private YouTube createMockYoutube() throws Exception {
- YouTube mockYouTube = mock(YouTube.class);
- YouTube.Channels channels = createMockChannels();
- when(mockYouTube.channels()).thenReturn(channels);
- return mockYouTube;
- }
-
- private YouTube.Channels createMockChannels() throws Exception {
- YouTube.Channels mockChannels = mock(YouTube.Channels.class);
- YouTube.Channels.List channelLists = createMockChannelsList();
- when(mockChannels.list(anyString())).thenReturn(channelLists);
- return mockChannels;
- }
-
- private YouTube.Channels.List createMockChannelsList() throws Exception {
- YouTube.Channels.List mockList = mock(YouTube.Channels.List.class);
- when(mockList.setId(anyString())).thenReturn(mockList);
- when(mockList.setKey(anyString())).thenReturn(mockList);
- ChannelListResponse response = createMockResponse();
- when(mockList.execute()).thenReturn(response);
- return mockList;
- }
-
- private ChannelListResponse createMockResponse() {
- ChannelListResponse response = new ChannelListResponse();
- List<Channel> channelList = new LinkedList<>();
- response.setItems(channelList);
- Channel channel = new Channel();
- channel.setId(ID);
- channelList.add(channel);
- return response;
- }
-
-}
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeProviderTest.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeProviderTest.java
deleted file mode 100644
index 9a7e547..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeProviderTest.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
-import org.apache.streams.youtube.YoutubeConfiguration;
-
-import com.google.api.services.youtube.YouTube;
-import org.joda.time.DateTime;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-
-/**
- * Test for YoutubeProvider.
- */
-public class YoutubeProviderTest {
-
- /**
- * Test that every collector will be run and that data queued from the collectors will be processed.
- */
- @Test
- public void testDataCollectorRunsPerUser() {
- Random random = new Random(System.currentTimeMillis());
- int numUsers = random.nextInt(1000);
- List<UserInfo> userList = new LinkedList<>();
-
- for ( int i = 0; i < numUsers; ++i ) {
- userList.add(new UserInfo());
- }
-
- YoutubeConfiguration config = new YoutubeConfiguration();
- config.setYoutubeUsers(userList);
- config.setApiKey("API_KEY");
- YoutubeProvider provider = buildProvider(config);
-
- try {
- provider.prepare(null);
- provider.startStream();
- int datumCount = 0;
- while (provider.isRunning()) {
- datumCount += provider.readCurrent().size();
- }
- assertEquals(numUsers, datumCount);
- } finally {
- provider.cleanUp();
- }
- }
-
- @Test
- public void testConfigSetterGetter() {
- YoutubeConfiguration config = new YoutubeConfiguration();
- config.setApiKey("API_KEY");
- config.setVersion("fake_version_1");
- YoutubeConfiguration newConfig = new YoutubeConfiguration();
- newConfig.setApiKey("API_KEY");
- config.setVersion("fake_version_2");
-
- YoutubeProvider provider = buildProvider(config);
-
- assertEquals(provider.getConfig(), config);
-
- provider.setConfig(newConfig);
- assertEquals(provider.getConfig(), newConfig);
- }
-
- @Test
- public void testUserInfoWithDefaultDates() {
- YoutubeConfiguration config = new YoutubeConfiguration();
- config.setApiKey("API_KEY");
- YoutubeProvider provider = buildProvider(config);
-
- DateTime afterDate = new DateTime(System.currentTimeMillis());
- DateTime beforeDate = afterDate.minus(10000);
-
- provider.setDefaultAfterDate(afterDate);
- provider.setDefaultBeforeDate(beforeDate);
-
- Set<String> users = new HashSet<>();
- users.add("test_user_1");
- users.add("test_user_2");
- users.add("test_user_3");
-
- provider.setUserInfoWithDefaultDates(users);
-
- List<UserInfo> youtubeUsers = provider.getConfig().getYoutubeUsers();
-
- for (UserInfo user : youtubeUsers) {
- assert (user.getAfterDate().equals(afterDate));
- assert (user.getBeforeDate().equals(beforeDate));
- }
- }
-
- @Test
- public void testUserInfoWithAfterDate() {
- YoutubeConfiguration config = new YoutubeConfiguration();
- config.setApiKey("API_KEY");
- YoutubeProvider provider = buildProvider(config);
-
- Map<String, DateTime> users = new HashMap<>();
- users.put("user1", new DateTime(System.currentTimeMillis()));
- users.put("user3", new DateTime(System.currentTimeMillis()));
- users.put("user4", new DateTime(System.currentTimeMillis()));
-
- provider.setUserInfoWithAfterDate(users);
-
- List<UserInfo> youtubeUsers = provider.getConfig().getYoutubeUsers();
-
- for (UserInfo user : youtubeUsers) {
- assert (user.getAfterDate().equals(users.get(user.getUserId())));
- }
- }
-
- private YoutubeProvider buildProvider(YoutubeConfiguration config) {
- return new YoutubeProvider(config) {
-
- @Override
- protected YouTube createYouTubeClient() throws IOException {
- return mock(YouTube.class);
- }
-
- @Override
- protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo) {
- final BlockingQueue<StreamsDatum> q = queue;
- return () -> {
- try {
- q.put(new StreamsDatum(null));
- } catch (InterruptedException ie) {
- fail("Test was interrupted");
- }
- };
- }
- };
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollectorTest.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollectorTest.java
deleted file mode 100644
index 1d92a3e..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/provider/YoutubeUserActivityCollectorTest.java
+++ /dev/null
@@ -1,354 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.provider;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.google.gplus.configuration.UserInfo;
-import org.apache.streams.local.queues.ThroughputQueue;
-import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
-import org.apache.streams.youtube.YoutubeConfiguration;
-
-import com.google.api.services.youtube.YouTube;
-import com.google.api.services.youtube.model.Activity;
-import com.google.api.services.youtube.model.ActivityContentDetails;
-import com.google.api.services.youtube.model.ActivityContentDetailsUpload;
-import com.google.api.services.youtube.model.ActivityListResponse;
-import com.google.api.services.youtube.model.Video;
-import com.google.api.services.youtube.model.VideoListResponse;
-import com.google.api.services.youtube.model.VideoSnippet;
-import org.joda.time.DateTime;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.anyLong;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Test for YoutubeUserActivityCollector.
- */
-public class YoutubeUserActivityCollectorTest {
- private static final String USER_ID = "fake_user_id";
- private static final String IN_RANGE_IDENTIFIER = "data in range";
- private YoutubeConfiguration config;
-
- private static VideoListResponse createMockVideoListResponse(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before, boolean page) {
- VideoListResponse feed = new VideoListResponse();
- List<Video> list = new LinkedList<>();
-
- for (int i = 0; i < numAfter; ++i) {
- com.google.api.client.util.DateTime published = new com.google.api.client.util.DateTime(after.getMillis() + 1000000);
- Video video = new Video();
- video.setSnippet(new VideoSnippet());
- video.getSnippet().setPublishedAt(published);
- list.add(video);
- }
- for (int i = 0; i < numInRange; ++i) {
- DateTime published;
- if ((before == null && after == null) || before == null) {
- published = DateTime.now(); // no date range or end time date range so just make the time now.
- } else if (after == null) {
- published = before.minusMillis(100000); //no beginning to range
- } else { // has to be in range
- long range = before.getMillis() - after.getMillis();
- published = after.plus(range / 2); //in the middle
- }
- com.google.api.client.util.DateTime ytPublished = new com.google.api.client.util.DateTime(published.getMillis());
- Video video = new Video();
- video.setSnippet(new VideoSnippet());
- video.getSnippet().setPublishedAt(ytPublished);
- video.getSnippet().setTitle(IN_RANGE_IDENTIFIER);
- list.add(video);
- }
- for (int i = 0; i < numBefore; ++i) {
- com.google.api.client.util.DateTime published = new com.google.api.client.util.DateTime((after.minusMillis(100000)).getMillis());
- Video video = new Video();
- video.setSnippet(new VideoSnippet());
- video.getSnippet().setPublishedAt(published);
- list.add(video);
- }
- if (page) {
- feed.setNextPageToken("A");
- } else {
- feed.setNextPageToken(null);
- }
-
- feed.setItems(list);
-
- return feed;
- }
-
- @Before
- public void setup() {
- this.config = new YoutubeConfiguration();
- this.config.setApiKey("API_KEY");
- }
-
- @Test
- public void testGetVideos() throws IOException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(0, 1, 0, now, now.minus(10000));
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- List<Video> video = collector.getVideoList("fake_video_id");
-
- assertNotNull(video.get(0));
- }
-
- @Test
- public void testGetVideosNull() throws IOException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(0, 0, 0, now.plus(10000), now.minus(10000));
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- List<Video> video = collector.getVideoList("fake_video_id");
-
- assertEquals(video.size(), 0);
- }
-
- @Test
- public void testProcessActivityFeed() throws IOException, InterruptedException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(0, 0, 5, now.plus(3000000), now.minus(1000000));
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- ActivityListResponse feed = buildActivityListResponse(1);
-
- collector.processActivityFeed(feed, new DateTime(System.currentTimeMillis()), null);
-
- assertEquals(collector.getDatumQueue().size(), 5);
- }
-
- @Test
- public void testProcessActivityFeedBefore() throws IOException, InterruptedException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(5, 0, 0, now, now);
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- ActivityListResponse feed = buildActivityListResponse(1);
-
- collector.processActivityFeed(feed, new DateTime(System.currentTimeMillis()), null);
-
- assertEquals(collector.getDatumQueue().size(), 0);
- }
-
- @Test
- public void testProcessActivityFeedAfter() throws IOException, InterruptedException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(0, 5, 0, now, now);
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- ActivityListResponse feed = buildActivityListResponse(1);
-
- collector.processActivityFeed(feed, new DateTime(now.getMillis() - 100000), null);
-
- assertEquals(collector.getDatumQueue().size(), 5);
- }
-
- @Test
- public void testProcessActivityFeedMismatchCount() throws IOException, InterruptedException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(5, 5, 5, now, now.minus(100000));
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- ActivityListResponse feed = buildActivityListResponse(1);
-
- collector.processActivityFeed(feed, new DateTime(now), null);
-
- assertEquals(collector.getDatumQueue().size(), 5);
- }
-
- @Test
- public void testProcessActivityFeedMismatchCountInRange() throws IOException, InterruptedException {
- DateTime now = new DateTime(System.currentTimeMillis());
- YouTube youtube = buildYouTube(5, 5, 5, now, now.minus(100000));
-
- BlockingQueue<StreamsDatum> datumQueue = new ThroughputQueue<>();
- YoutubeUserActivityCollector collector = new YoutubeUserActivityCollector(youtube, datumQueue, new ExponentialBackOffStrategy(2), new UserInfo().withUserId(USER_ID), this.config);
-
- ActivityListResponse feed = buildActivityListResponse(1);
-
- collector.processActivityFeed(feed, new DateTime(now), new DateTime(now).minus(100000));
-
- assertEquals(collector.getDatumQueue().size(), 5);
- }
-
- private ActivityListResponse buildActivityListResponse(int num) {
- ActivityListResponse activityListResponse = new ActivityListResponse();
- List<Activity> items = new ArrayList<>();
-
- for ( int x = 0; x < num; x++ ) {
- Activity activity = new Activity();
-
- ActivityContentDetails contentDetails = new ActivityContentDetails();
- ActivityContentDetailsUpload upload = new ActivityContentDetailsUpload();
- upload.setVideoId("video_id_" + x);
- contentDetails.setUpload(upload);
-
- activity.setId("id_" + x);
- activity.setContentDetails(contentDetails);
-
- items.add(activity);
- }
-
- activityListResponse.setItems(items);
-
- return activityListResponse;
- }
-
- private YouTube buildYouTube(int numBeforeRange, int numAfterRange, int numInRange, DateTime afterDate, DateTime beforeDate) {
-
- return createYoutubeMock(numBeforeRange, numAfterRange, numInRange, afterDate, beforeDate);
-
- }
-
- private YouTube createYoutubeMock(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
- YouTube youtube = mock(YouTube.class);
-
- final YouTube.Videos videos = createMockVideos(numBefore, numAfter, numInRange, after, before);
- doAnswer(invocationOnMock -> videos).when(youtube).videos();
-
- return youtube;
- }
-
- private YouTube.Videos createMockVideos(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
- YouTube.Videos videos = mock(YouTube.Videos.class);
-
- try {
- YouTube.Videos.List list = createMockVideosList(numBefore, numAfter, numInRange, after, before);
- when(videos.list(anyString())).thenReturn(list);
- } catch (IOException ex) {
- fail("Exception thrown while creating mock");
- }
-
- return videos;
- }
-
- private YouTube.Videos.List createMockVideosList(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
- YouTube.Videos.List list = mock(YouTube.Videos.List.class);
-
- when(list.setMaxResults(anyLong())).thenReturn(list);
- when(list.setPageToken(anyString())).thenReturn(list);
- when(list.setId(anyString())).thenReturn(list);
- when(list.setKey(anyString())).thenReturn(list);
-
- VideoListResponseAnswer answer = new VideoListResponseAnswer(numBefore, numAfter, numInRange, after, before);
- try {
- doAnswer(answer).when(list).execute();
- } catch (IOException ioe) {
- fail("Should not have thrown exception while creating mock. : " + ioe.getMessage());
- }
-
- return list;
- }
-
- private static class VideoListResponseAnswer implements Answer<VideoListResponse> {
- private int afterCount = 0;
- private int beforeCount = 0;
- private int inCount = 0;
- private int maxBatch = 100;
-
- private int numAfter;
- private int numInRange;
- private int numBefore;
- private DateTime after;
- private DateTime before;
-
- private VideoListResponseAnswer(int numBefore, int numAfter, int numInRange, DateTime after, DateTime before) {
- this.numBefore = numBefore;
- this.numAfter = numAfter;
- this.numInRange = numInRange;
- this.after = after;
- this.before = before;
- }
-
- @Override
- public VideoListResponse answer(InvocationOnMock invocationOnMock) throws Throwable {
- int totalCount = 0;
- int batchAfter = 0;
- int batchBefore = 0;
- int batchIn = 0;
- inCount = 0;
- afterCount = 0;
- beforeCount = 0;
-
- if (afterCount != numAfter) {
- if (numAfter - afterCount >= maxBatch) {
- afterCount += maxBatch;
- batchAfter += maxBatch;
- totalCount += batchAfter;
- } else {
- batchAfter += numAfter - afterCount;
- totalCount += numAfter - afterCount;
- afterCount = numAfter;
- }
- }
- if (totalCount < maxBatch && inCount != numInRange) {
- if (numInRange - inCount >= maxBatch - totalCount) {
- inCount += maxBatch - totalCount;
- batchIn += maxBatch - totalCount;
- totalCount += batchIn;
- } else {
- batchIn += numInRange - inCount;
- totalCount += numInRange - inCount;
- inCount = numInRange;
- }
- }
- if (totalCount < maxBatch && beforeCount != numBefore) {
- if (numBefore - batchBefore >= maxBatch - totalCount) {
- batchBefore += maxBatch - totalCount;
- totalCount = maxBatch;
- beforeCount += batchBefore;
- } else {
- batchBefore += numBefore - beforeCount;
- totalCount += numBefore - beforeCount;
- beforeCount = numBefore;
- }
- }
-
- return createMockVideoListResponse(batchBefore, batchAfter, batchIn, after, before, numAfter != afterCount || inCount != numInRange || beforeCount != numBefore);
- }
- }
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeEventClassifierTest.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeEventClassifierTest.java
deleted file mode 100644
index d473abe..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeEventClassifierTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.serializer;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.api.services.youtube.model.Video;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class YoutubeEventClassifierTest {
-
- private final String testVideo = "{\"etag\":\"\\\"4FSIjSQU83ZJMYAO0IqRYMvZX98/V0q3OIauZ3ZAkszLUDbHL45yEGM\\\"\",\"id\":\"sUOepRctwVE\",\"kind\":\"youtube#video\",\"snippet\":{\"channelId\":\"UCNENOn2nmwguQYkejKhJGPQ\",\"channelTitle\":\"Carilion Clinic\",\"description\":\"Join Carilion Clinic's Heart Failure experts for a LIVE Google+ Hangout on Feb. 23, 12:30-1 p.m. to learn more about heart failure, treatment options, and lifestyle changes. Learn more: https://plus.google.com/u/0/eve [...]
- private final String testObjectNode = "{\"etag\":\"\\\"4FSIjSQU83ZJMYAO0IqRYMvZX98/V0q3OIauZ3ZAkszLUDbHL45yEGM\\\"\",\"id\":\"sUOepRctwVE\",\"kind\":\"youtube#somethingElse\",\"snippet\":{\"channelId\":\"UCNENOn2nmwguQYkejKhJGPQ\",\"channelTitle\":\"Carilion Clinic\",\"description\":\"Join Carilion Clinic's Heart Failure experts for a LIVE Google+ Hangout on Feb. 23, 12:30-1 p.m. to learn more about heart failure, treatment options, and lifestyle changes. Learn more: https://plus.googl [...]
-
- @Test
- public void testVideoClassification() {
- Class klass = YoutubeEventClassifier.detectClass(testVideo);
-
- assertEquals(klass, Video.class);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testExceptionClassification() {
- YoutubeEventClassifier.detectClass("");
- }
-
- @Test
- public void testObjectNodeClassification() {
- Class klass = YoutubeEventClassifier.detectClass(testObjectNode);
-
- assertEquals(klass, ObjectNode.class);
- }
-}
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeVideoSerDeTest.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeVideoSerDeTest.java
deleted file mode 100644
index e4affad..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/serializer/YoutubeVideoSerDeTest.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.serializer;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Provider;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.api.services.youtube.model.Video;
-import org.joda.time.DateTime;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for YoutubeVideoSerDe.
- */
-public class YoutubeVideoSerDeTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeVideoSerDeTest.class);
- private final String testVideo = "{\"etag\":\"\\\"4FSIjSQU83ZJMYAO0IqRYMvZX98/V0q3OIauZ3ZAkszLUDbHL45yEGM\\\"\",\"id\":\"sUOepRctwVE\",\"kind\":\"youtube#video\",\"snippet\":{\"channelId\":\"UCNENOn2nmwguQYkejKhJGPQ\",\"channelTitle\":\"Carilion Clinic\",\"description\":\"Join Carilion Clinic's Heart Failure experts for a LIVE Google+ Hangout on Feb. 23, 12:30-1 p.m. to learn more about heart failure, treatment options, and lifestyle changes. Learn more: https://plus.google.com/u/0/eve [...]
- private ObjectMapper objectMapper;
-
- /**
- * setup for test.
- */
- @Before
- public void setup() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(Video.class, new YoutubeVideoDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- @Test
- public void testVideoObject() {
- LOGGER.info("raw: {}", testVideo);
-
- try {
- Activity activity = new Activity();
-
- Video video = objectMapper.readValue(testVideo, Video.class);
-
- YoutubeActivityUtil.updateActivity(video, activity, "testChannelId");
- LOGGER.info("activity: {}", activity);
-
- assertNotNull(activity);
- assert (activity.getId().contains("id:youtube:post"));
- assertEquals(activity.getVerb(), "post");
-
- Provider provider = activity.getProvider();
- assertEquals(provider.getId(), "id:providers:youtube");
- assertEquals(provider.getDisplayName(), "YouTube");
-
- ActivityObject actor = activity.getActor();
- assert (actor.getId().contains("id:youtube:"));
- assertNotNull(actor.getDisplayName());
- assertNotNull(actor.getSummary());
-
- assertNotNull(activity.getTitle());
- assertNotNull(activity.getUrl());
- assertNotNull(activity.getContent());
-
- assertEquals(activity.getPublished().getClass(), DateTime.class);
-
- Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity);
-
- assertNotNull(extensions.get("youtube"));
- assertNotNull(extensions.get("likes"));
-
- assertTrue(testActivityObject(activity));
- } catch (Exception ex) {
- LOGGER.error("Exception while testing the Ser/De functionality of the Video deserializer: {}", ex);
- }
- }
-
- private boolean testActivityObject(Activity activity) {
- boolean valid = false;
-
- ActivityObject obj = activity.getObject();
-
- if ( obj.getObjectType().equals("video")
- && !(obj.getImage() == null)
- && !obj.getUrl().equals("null")
- && obj.getUrl().contains("https://www.youtube.com/watch?v=")) {
- valid = true;
- }
-
- return valid;
- }
-}
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeChannelProviderIT.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeChannelProviderIT.java
deleted file mode 100644
index fac0b2e..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeChannelProviderIT.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.test.providers;
-
-import org.apache.streams.youtube.provider.YoutubeChannelProvider;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-import static org.testng.Assert.assertTrue;
-
-/**
- * YoutubeChannelProviderIT integration test for YoutubeChannelProvider.
- */
-public class YoutubeChannelProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeChannelProviderIT.class);
-
- @Test
- public void testYoutubeChannelProvider() throws Exception {
-
- String configfile = "./target/test-classes/YoutubeChannelProviderIT.conf";
- String outfile = "./target/test-classes/YoutubeChannelProviderIT.stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- YoutubeChannelProvider.main(args);
- } catch ( Exception ex ) {
- LOGGER.error("Test Exception!", ex);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- File out = new File(outfile);
- assertTrue (out.exists());
- assertTrue (out.canRead());
- assertTrue (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- assertTrue (outCounter.getLineNumber() >= 1);
-
- }
-}
diff --git a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeUserActivityProviderIT.java b/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeUserActivityProviderIT.java
deleted file mode 100644
index df1eaef..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/java/org/apache/streams/youtube/test/providers/YoutubeUserActivityProviderIT.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.youtube.test.providers;
-
-import org.apache.streams.youtube.provider.YoutubeUserActivityProvider;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.LineNumberReader;
-
-/**
- * Integration Test for YoutubeUserActivityProvider.
- */
-public class YoutubeUserActivityProviderIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeUserActivityProviderIT.class);
-
- @Test
- public void testYoutubeUserActivityProvider() throws Exception {
-
- String configfile = "./target/test-classes/YoutubeUserActivityProviderIT.conf";
- String outfile = "./target/test-classes/YoutubeUserActivityProviderIT.stdout.txt";
-
- String[] args = new String[2];
- args[0] = configfile;
- args[1] = outfile;
-
- Thread testThread = new Thread(() -> {
- try {
- YoutubeUserActivityProvider.main(args);
- } catch ( Exception ex ) {
- LOGGER.error("Test Exception!", ex);
- }
- });
- testThread.start();
- testThread.join(60000);
-
- File out = new File(outfile);
- Assert.assertTrue (out.exists());
- Assert.assertTrue (out.canRead());
- Assert.assertTrue (out.isFile());
-
- FileReader outReader = new FileReader(out);
- LineNumberReader outCounter = new LineNumberReader(outReader);
-
- while (outCounter.readLine() != null) {}
-
- Assert.assertTrue(outCounter.getLineNumber() >= 250);
-
- }
-}
diff --git a/streams-contrib/streams-provider-youtube/src/test/resources/YoutubeChannelProviderIT.conf b/streams-contrib/streams-provider-youtube/src/test/resources/YoutubeChannelProviderIT.conf
deleted file mode 100644
index 8565c45..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/resources/YoutubeChannelProviderIT.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-youtube {
- youtubeUsers = [
- {
- userId = "UCLDJ_V9KUOdOFSbDvPfGBxw"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-contrib/streams-provider-youtube/src/test/resources/YoutubeUserActivityProviderIT.conf b/streams-contrib/streams-provider-youtube/src/test/resources/YoutubeUserActivityProviderIT.conf
deleted file mode 100644
index 8565c45..0000000
--- a/streams-contrib/streams-provider-youtube/src/test/resources/YoutubeUserActivityProviderIT.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-youtube {
- youtubeUsers = [
- {
- userId = "UCLDJ_V9KUOdOFSbDvPfGBxw"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-core/README.md b/streams-core/README.md
deleted file mode 100644
index 2826219..0000000
--- a/streams-core/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-core
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-core/pom.xml b/streams-core/pom.xml
deleted file mode 100644
index b73fbfc..0000000
--- a/streams-core/pom.xml
+++ /dev/null
@@ -1,83 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>apache-streams</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-core</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Core classes and interfaces for streams.</description>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>javax.ws.rs</groupId>
- <artifactId>jsr311-api</artifactId>
- <version>1.1.1</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- </build>
-</project>
diff --git a/streams-core/src/main/java/org/apache/streams/core/DatumStatus.java b/streams-core/src/main/java/org/apache/streams/core/DatumStatus.java
deleted file mode 100644
index 5846665..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/DatumStatus.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.core;
-
-/**
- * Status of StreamsDatum.
- */
-public enum DatumStatus {
- SUCCESS,
- PARTIAL,
- FAIL
-}
diff --git a/streams-core/src/main/java/org/apache/streams/core/DatumStatusCountable.java b/streams-core/src/main/java/org/apache/streams/core/DatumStatusCountable.java
deleted file mode 100644
index c00e378..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/DatumStatusCountable.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.core;
-
-/**
- * Created by steveblackmon on 3/24/14.
- */
-@Deprecated
-public interface DatumStatusCountable {
-
- @Deprecated
- public DatumStatusCounter getDatumStatusCounter();
-
-}
diff --git a/streams-core/src/main/java/org/apache/streams/core/DatumStatusCounter.java b/streams-core/src/main/java/org/apache/streams/core/DatumStatusCounter.java
deleted file mode 100644
index 250cc7e..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/DatumStatusCounter.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.core;
-
-import java.io.Serializable;
-
-@Deprecated
-public class DatumStatusCounter implements Serializable {
-
- private volatile int attempted = 0;
- private volatile int success = 0;
- private volatile int fail = 0;
- private volatile int partial = 0;
- private volatile int emitted = 0;
-
- public int getAttempted() {
- return this.attempted;
- }
-
- public int getSuccess() {
- return this.success;
- }
-
- public int getFail() {
- return this.fail;
- }
-
- public int getPartial() {
- return this.partial;
- }
-
- public int getEmitted() {
- return this.emitted;
- }
-
- public DatumStatusCounter() {
- }
-
- /**
- * accumulate partial DatumStatusCounter.
- * @param datumStatusCounter DatumStatusCounter
- */
- @Deprecated
- public void add(DatumStatusCounter datumStatusCounter) {
- this.attempted += datumStatusCounter.getAttempted();
- this.success += datumStatusCounter.getSuccess();
- this.partial = datumStatusCounter.getPartial();
- this.fail += datumStatusCounter.getFail();
- this.emitted += datumStatusCounter.getEmitted();
- }
-
- @Deprecated
- public void incrementAttempt() {
- this.attempted += 1;
- }
-
- @Deprecated
- public void incrementAttempt(int counter) {
- this.attempted += counter;
- }
-
- /**
- * increment specific DatumStatus by 1.
- * @param workStatus DatumStatus
- */
- @Deprecated
- public synchronized void incrementStatus(DatumStatus workStatus) {
- // add this to the record counter
- switch (workStatus) {
- case SUCCESS:
- this.success++;
- break;
- case PARTIAL:
- this.partial++;
- break;
- case FAIL:
- this.fail++;
- break;
- default:
- break;
- }
- this.emitted += 1;
- }
-
- /**
- * increment specific DatumStatus by count.
- * @param workStatus DatumStatus
- * @param counter counter
- */
- @Deprecated
- public synchronized void incrementStatus(DatumStatus workStatus, int counter) {
- // add this to the record counter
- switch (workStatus) {
- case SUCCESS:
- this.success += counter;
- break;
- case PARTIAL:
- this.partial += counter;
- break;
- case FAIL:
- this.fail += counter;
- break;
- default:
- break;
- }
- this.emitted += counter;
- }
-
- @Override
- public String toString() {
- return "DatumStatusCounter{"
- + "attempted=" + attempted
- + ", success=" + success
- + ", fail=" + fail
- + ", partial=" + partial
- + ", emitted=" + emitted
- + '}';
- }
-}
diff --git a/streams-core/src/main/java/org/apache/streams/core/StreamBuilder.java b/streams-core/src/main/java/org/apache/streams/core/StreamBuilder.java
deleted file mode 100644
index 0920829..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/StreamBuilder.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.core;
-
-import org.apache.streams.config.StreamsConfiguration;
-
-import org.joda.time.DateTime;
-
-import java.io.Serializable;
-import java.math.BigInteger;
-
-/**
- * Interface for building data streams.
- *
- * <pre>
- * StreamBuilder builder = ...
- * builder.newReadCurrentStream(. . .)
- * .addStreamsProcessor(. . .)
- * ...
- * .addStreamsPersistWriter(. . .)
- * builder.run();
- * </pre>
- *
- */
-public interface StreamBuilder extends Serializable {
-
- StreamBuilder setStreamsConfiguration(StreamsConfiguration configuration);
-
- StreamsConfiguration getStreamsConfiguration();
-
- /**
- * Add a {@link org.apache.streams.core.StreamsProcessor} to the data processing stream.
- * @param processorId unique id for this processor - must be unique across the entire stream
- * @param processor the processor to execute
- * @param numTasks the number of instances of this processor to run concurrently
- * @param connectToIds the ids of the {@link org.apache.streams.core.StreamsOperation} that this process will
- * receive data from.
- * @return this
- */
- StreamBuilder addStreamsProcessor(String processorId, StreamsProcessor processor, int numTasks, String... connectToIds);
-
- /**
- * Add a {@link org.apache.streams.core.StreamsPersistWriter} to the data processing stream.
- * @param persistWriterId unique id for this processor - must be unique across the entire stream
- * @param writer the writer to execute
- * @param numTasks the number of instances of this writer to run concurrently
- * @param connectToIds the ids of the {@link org.apache.streams.core.StreamsOperation} that this process will
- * receive data from.
- * @return this
- */
- StreamBuilder addStreamsPersistWriter(String persistWriterId, StreamsPersistWriter writer, int numTasks, String... connectToIds);
-
- /**
- * Add a {@link org.apache.streams.core.StreamsProvider} to the data processing stream. The provider will execute
- * {@link org.apache.streams.core.StreamsProvider:readCurrent()} to produce data.
- * @param streamId unique if for this provider - must be unique across the entire stream.
- * @param provider provider to execute
- * @return this
- */
- StreamBuilder newPerpetualStream(String streamId, StreamsProvider provider);
-
- /**
- * Add a {@link org.apache.streams.core.StreamsProvider} to the data processing stream. The provider will execute
- * {@link org.apache.streams.core.StreamsProvider:readCurrent()} to produce data.
- * @param streamId unique if for this provider - must be unique across the entire stream.
- * @param provider provider to execute
- * @return this
- */
- StreamBuilder newReadCurrentStream(String streamId, StreamsProvider provider);
-
- /**
- * Add a {@link org.apache.streams.core.StreamsProvider} to the data processing stream. The provider will execute
- * {@link org.apache.streams.core.StreamsProvider:readNext(BigInteger)} to produce data.
- * @param streamId unique if for this provider - must be unique across the entire stream.
- * @param provider provider to execute
- * @param sequence sequence to pass to {@link org.apache.streams.core.StreamsProvider:readNext(BigInteger)} method
- * @return this
- */
- StreamBuilder newReadNewStream(String streamId, StreamsProvider provider, BigInteger sequence);
-
- /**
- * Add a {@link org.apache.streams.core.StreamsProvider} to the data processing stream. The provider will execute
- * {@link org.apache.streams.core.StreamsProvider:readRange(DateTime, DateTime)} to produce data. Whether the start
- * and end dates are inclusive or exclusive is up to the implementation.
- * @param streamId unique if for this provider - must be unique across the entire stream.
- * @param provider provider to execute
- * @param start start date
- * @param end end date
- * @return this
- */
- StreamBuilder newReadRangeStream(String streamId, StreamsProvider provider, DateTime start, DateTime end);
-
- /**
- * Builds the stream, and starts it or submits it based on implementation.
- */
- void start();
-
- /**
- * Stops the streams processing. No guarantee on a smooth shutdown. Optional method, may not be implemented in
- * all cases.
- */
- void stop();
-
-}
diff --git a/streams-core/src/main/java/org/apache/streams/core/StreamsDatum.java b/streams-core/src/main/java/org/apache/streams/core/StreamsDatum.java
deleted file mode 100644
index 27f3d85..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/StreamsDatum.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.core;
-
-import org.apache.streams.pojo.json.Activity;
-
-import org.joda.time.DateTime;
-
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.HashMap;
-import java.util.Map;
-
-public class StreamsDatum implements Serializable {
-
- public StreamsDatum(Object document) {
- this(document, null, null, null, new HashMap<>());
- }
-
- public StreamsDatum(Object document, String id) {
- this(document, id, null, null, new HashMap<>());
- }
-
- public StreamsDatum(Object document, BigInteger sequenceid) {
- this(document, null, null, sequenceid);
- }
-
- public StreamsDatum(Object document, DateTime timestamp) {
- this(document, null, timestamp, null);
- }
-
- public StreamsDatum(Object document, DateTime timestamp, BigInteger sequenceid) {
- this(document, null, timestamp, sequenceid);
- }
-
- public StreamsDatum(Object document, DateTime timestamp, Map<String, Object> metadata) {
- this(document, null, timestamp, null, metadata);
- }
-
- public StreamsDatum(Object document, String id, DateTime timestamp) {
- this(document, id, timestamp, null, new HashMap<>());
- }
-
- public StreamsDatum(Object document, String id, Map<String, Object> metadata) {
- this(document, id, null, null, metadata);
- }
-
- public StreamsDatum(Object document, String id, BigInteger sequenceid, Map<String, Object> metadata) {
- this(document, id, null, sequenceid, metadata);
- }
-
- public StreamsDatum(Object document, String id, BigInteger sequenceid) {
- this(document, id, sequenceid, new HashMap<>());
- }
-
- public StreamsDatum(Object document, String id, DateTime timestamp, BigInteger sequenceid) {
- this.document = document;
- this.id = id;
- this.timestamp = timestamp;
- this.sequenceid = sequenceid;
- this.metadata = new HashMap<>();
- }
-
- public StreamsDatum(Object document, String id, DateTime timestamp, BigInteger sequenceid, Map<String, Object> metadata) {
- this.document = document;
- this.id = id;
- this.timestamp = timestamp;
- this.sequenceid = sequenceid;
- this.metadata = metadata;
- }
-
- public DateTime timestamp;
-
- public BigInteger sequenceid;
-
- public Map<String, Object> metadata;
-
- public Object document;
-
- private String id;
-
- public DateTime getTimestamp() {
- return timestamp;
- }
-
- public void setTimestamp(DateTime timestamp) {
- this.timestamp = timestamp;
- }
-
- public BigInteger getSequenceid() {
- return sequenceid;
- }
-
- public void setSequenceid(BigInteger sequenceid) {
- this.sequenceid = sequenceid;
- }
-
- public Map<String, Object> getMetadata() {
- return metadata;
- }
-
- public void setMetadata(Map<String, Object> metadata) {
- this.metadata = metadata;
- }
-
- public Object getDocument() {
- return document;
- }
-
- public void setDocument(Object document) {
- this.document = document;
- }
-
- public String getId() {
- if (this.id == null && this.document instanceof Activity) {
- return ((Activity)this.document).getId();
- }
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- @Override
- public boolean equals(Object configurationObject) {
- if (configurationObject instanceof StreamsDatum) {
- StreamsDatum that = (StreamsDatum) configurationObject;
- if (this.document != null && this.document.equals(that.document)) {
- return (this.timestamp != null ? this.timestamp.equals(that.timestamp) : that.timestamp == null)
- && (this.sequenceid != null ? this.sequenceid.equals(that.sequenceid) : that.sequenceid == null);
- } else {
- return that.document == null && this.document == null;
- }
- } else {
- return false;
- }
- }
-
- @Override
- public String toString() {
- return "StreamsDatum{"
- + "timestamp=" + timestamp
- + ", sequenceid=" + sequenceid
- + ", metadata=" + metadata
- + ", document=" + document
- + ", id='" + id + '\''
- + '}';
- }
-
-}
diff --git a/streams-core/src/main/java/org/apache/streams/core/StreamsOperation.java b/streams-core/src/main/java/org/apache/streams/core/StreamsOperation.java
deleted file mode 100644
index a86499b..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/StreamsOperation.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.core;
-
-import java.io.Serializable;
-
-/**
- * Each step in a pipeline is a StreamsOperation.
- */
-public interface StreamsOperation extends Serializable {
-
- /**
- * Each operation must publish an identifier.
- */
- String getId();
-
- /**
- * This method will be called after initialization/serialization. Initialize any non-serializable objects here.
- * @param configurationObject Any object to help intialize the operation. ie. Map, JobContext, Properties, etc. The type
- * will be based on where the operation is being run (ie. hadoop, storm, locally, etc.)
- */
- void prepare(Object configurationObject);
-
- /**
- * No guarantee that this method will ever be called. But upon shutdown of the stream, an attempt to call this method
- * will be made.
- * Use this method to terminate connections, etc.
- */
- void cleanUp();
-}
diff --git a/streams-core/src/main/java/org/apache/streams/core/StreamsPersistReader.java b/streams-core/src/main/java/org/apache/streams/core/StreamsPersistReader.java
deleted file mode 100644
index 64063ac..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/StreamsPersistReader.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.core;
-
-import org.joda.time.DateTime;
-
-import java.math.BigInteger;
-
-/**
- * Currently a duplicate interface. Has exact same methods as StreamsProvider.
- */
-public interface StreamsPersistReader extends StreamsProvider {
-
- StreamsResultSet readAll();
-
- StreamsResultSet readNew(BigInteger sequence);
-
- StreamsResultSet readRange(DateTime start, DateTime end);
-
-}
diff --git a/streams-core/src/main/java/org/apache/streams/core/StreamsPersistWriter.java b/streams-core/src/main/java/org/apache/streams/core/StreamsPersistWriter.java
deleted file mode 100644
index 59797e4..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/StreamsPersistWriter.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.core;
-
-/**
- * StreamsOperation for writing data out of a pipeline.
- */
-public interface StreamsPersistWriter extends StreamsOperation {
-
- /**
- * Persist the StreamsDatum to the corresponding data store.
- * @param entry to be stored.
- */
- void write( StreamsDatum entry );
-
-}
diff --git a/streams-core/src/main/java/org/apache/streams/core/StreamsProcessor.java b/streams-core/src/main/java/org/apache/streams/core/StreamsProcessor.java
deleted file mode 100644
index 25e1a07..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/StreamsProcessor.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.core;
-
-import java.util.List;
-
-public interface StreamsProcessor extends StreamsOperation {
-
- /**
- * Process/Analyze the {@link org.apache.streams.core.StreamsDatum} and return the the StreamsDatums that will
- * passed to every down stream operation that reads from this processor.
- * @param entry StreamsDatum to be processed
- * @return resulting StreamDatums from processing. Should never be null or contain null object. Empty list OK.
- */
- List<StreamsDatum> process( StreamsDatum entry );
-
-}
diff --git a/streams-core/src/main/java/org/apache/streams/core/StreamsProvider.java b/streams-core/src/main/java/org/apache/streams/core/StreamsProvider.java
deleted file mode 100644
index 2547343..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/StreamsProvider.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.core;
-
-import org.joda.time.DateTime;
-
-import java.math.BigInteger;
-
-/**
- * A StreamsProvider represents the entry point into the Streams pipeline. Providers are responsible for inserting
- * data into the pipeline in discrete result sets.
- */
-public interface StreamsProvider extends StreamsOperation {
-
- /**
- * Start the operation of the stream.
- */
- void startStream();
-
- /**
- * Read the current items available from the provider.
- * @return a non-null {@link org.apache.streams.core.StreamsResultSet}
- */
- StreamsResultSet readCurrent();
-
- /**
- * Read data with sequenceId greater than sequence.
- * @param sequence BigInteger sequence
- * @return {@link StreamsResultSet}
- */
- StreamsResultSet readNew(BigInteger sequence);
-
- /**
- * Read data with event time between start DateTime and end DateTime.
- * @param start start DateTime
- * @param end end DateTime
- * @return {@link StreamsResultSet}
- */
- StreamsResultSet readRange(DateTime start, DateTime end);
-
- /**
- * Flag to indicate whether the provider is still producing data.
- * @return true if the processor is actively awaiting or producing data. False otherwise.
- */
- boolean isRunning();
-}
-
diff --git a/streams-core/src/main/java/org/apache/streams/core/StreamsResultSet.java b/streams-core/src/main/java/org/apache/streams/core/StreamsResultSet.java
deleted file mode 100644
index d64f27e..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/StreamsResultSet.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.core;
-
-import java.util.Iterator;
-import java.util.Queue;
-
-/**
- * StreamsResultSet is a wrapper for an Iterator around a set of StreamsDatum.
- */
-public class StreamsResultSet implements Iterable<StreamsDatum> {
-
- Queue<StreamsDatum> queue;
-
- DatumStatusCounter counter;
-
- public StreamsResultSet(Queue<StreamsDatum> queue) {
- this.queue = queue;
- }
-
-
- @Override
- public Iterator<StreamsDatum> iterator() {
- return queue.iterator();
- }
-
- public int size() {
- return queue.size();
- }
-
- public Queue<StreamsDatum> getQueue() {
- return queue;
- }
-
- public void setQueue(Queue<StreamsDatum> queue) {
- this.queue = queue;
- }
-
- public DatumStatusCounter getCounter() {
- return counter;
- }
-
- public void setCounter(DatumStatusCounter counter) {
- this.counter = counter;
- }
-}
-
diff --git a/streams-core/src/main/java/org/apache/streams/core/util/DatumUtils.java b/streams-core/src/main/java/org/apache/streams/core/util/DatumUtils.java
deleted file mode 100644
index 7d80098..0000000
--- a/streams-core/src/main/java/org/apache/streams/core/util/DatumUtils.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.core.util;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsOperation;
-
-import org.joda.time.DateTime;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Provides common utilities for managing and manipulating StreamsDatums.
- */
-public class DatumUtils {
-
- /**
- * Adds an error occurred during a StreamsOperation step to the StreamsDatum's metadata. By convention, errors are
- * placed in the metadata under the "errors" and are organized by class simple name where the failure occurred.
- *
- * @param datum the datum on which the operation step errored
- * @param throwable the throwable encountered
- * @param operationClass the class of the operation
- */
- @SuppressWarnings("all")
- public static void addErrorToMetadata(StreamsDatum datum, Throwable throwable, Class<? extends StreamsOperation> operationClass) {
- if (!datum.getMetadata().containsKey("errors")) {
- datum.getMetadata().put("errors", new HashMap<String, Throwable>());
- }
- Map<String, Throwable> errors = (Map)datum.getMetadata().get("errors");
- errors.put(operationClass.getCanonicalName(), throwable);
- }
-
- public static StreamsDatum cloneDatum(StreamsDatum datum) {
- StreamsDatum clone = new StreamsDatum(datum.getDocument());
- clone.setId(datum.getId() == null ? null : datum.getId());
- clone.setTimestamp(datum.getTimestamp() == null ? null : new DateTime(datum.getTimestamp()));
- clone.setSequenceid(datum.getSequenceid() == null ? null : datum.getSequenceid());
- clone.setMetadata(datum.getMetadata() == null ? null : new HashMap<>(datum.getMetadata()));
- return clone;
- }
-}
diff --git a/streams-core/src/site/markdown/index.md b/streams-core/src/site/markdown/index.md
deleted file mode 100644
index f16365f..0000000
--- a/streams-core/src/site/markdown/index.md
+++ /dev/null
@@ -1,8 +0,0 @@
-streams-core
-==============
-
-streams-core contains core classes and interfaces.
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-monitoring/README.md b/streams-monitoring/README.md
deleted file mode 100644
index 2e35c70..0000000
--- a/streams-monitoring/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-streams-monitoring
-==============
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-monitoring/pom.xml b/streams-monitoring/pom.xml
deleted file mode 100644
index fb45135..0000000
--- a/streams-monitoring/pom.xml
+++ /dev/null
@@ -1,138 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>apache-streams</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-monitoring</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Monitoring of executing streams and stream components.</description>
-
- <dependencies>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.pojo.json</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes combine.children="append">
- <exclude>src/test/resources/MemoryUsageObjects.json</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-
-</project>
diff --git a/streams-monitoring/src/main/java/org/apache/streams/jackson/DatumStatusCounterDeserializer.java b/streams-monitoring/src/main/java/org/apache/streams/jackson/DatumStatusCounterDeserializer.java
deleted file mode 100644
index f13a44f..0000000
--- a/streams-monitoring/src/main/java/org/apache/streams/jackson/DatumStatusCounterDeserializer.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-import org.apache.streams.pojo.json.DatumStatusCounterBroadcast;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import org.slf4j.Logger;
-
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.util.Arrays;
-import javax.management.MBeanAttributeInfo;
-import javax.management.MBeanInfo;
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
-
-public class DatumStatusCounterDeserializer extends JsonDeserializer<DatumStatusCounterBroadcast> {
-
- private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(DatumStatusCounterDeserializer.class);
-
- public DatumStatusCounterDeserializer() {
-
- }
-
- @Override
- public DatumStatusCounterBroadcast deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
- try {
- MBeanServer server = ManagementFactory.getPlatformMBeanServer();
-
- DatumStatusCounterBroadcast datumStatusCounterBroadcast = new DatumStatusCounterBroadcast();
- JsonNode attributes = jsonParser.getCodec().readTree(jsonParser);
-
- ObjectName name = new ObjectName(attributes.get("canonicalName").asText());
- MBeanInfo info = server.getMBeanInfo(name);
- datumStatusCounterBroadcast.setName(name.toString());
-
- for (MBeanAttributeInfo attribute : Arrays.asList(info.getAttributes())) {
- try {
- switch (attribute.getName()) {
- case "Failed":
- datumStatusCounterBroadcast.setFailed((boolean) server.getAttribute(name, attribute.getName()));
- break;
- case "Passed":
- datumStatusCounterBroadcast.setPassed((boolean) server.getAttribute(name, attribute.getName()));
- break;
- default:
- break;
- }
- } catch (Exception ex) {
- LOGGER.error("Exception trying to deserialize DatumStatusCounterBroadcast object: {}", ex);
- }
- }
-
- return datumStatusCounterBroadcast;
- } catch (Exception ex) {
- LOGGER.error("Exception trying to deserialize DatumStatusCounterBroadcast object: {}", ex);
- return null;
- }
- }
-}
diff --git a/streams-monitoring/src/main/java/org/apache/streams/jackson/MemoryUsageDeserializer.java b/streams-monitoring/src/main/java/org/apache/streams/jackson/MemoryUsageDeserializer.java
deleted file mode 100644
index 8259340..0000000
--- a/streams-monitoring/src/main/java/org/apache/streams/jackson/MemoryUsageDeserializer.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-import org.apache.streams.pojo.json.MemoryUsageBroadcast;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import org.slf4j.Logger;
-
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.util.Arrays;
-import javax.management.MBeanAttributeInfo;
-import javax.management.MBeanInfo;
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
-import javax.management.openmbean.CompositeDataSupport;
-
-public class MemoryUsageDeserializer extends JsonDeserializer<MemoryUsageBroadcast> {
-
- private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(MemoryUsageDeserializer.class);
-
- public MemoryUsageDeserializer() {
-
- }
-
- @Override
- public MemoryUsageBroadcast deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
- try {
- MBeanServer server = ManagementFactory.getPlatformMBeanServer();
-
- MemoryUsageBroadcast memoryUsageBroadcast = new MemoryUsageBroadcast();
- JsonNode attributes = jsonParser.getCodec().readTree(jsonParser);
-
- ObjectName name = new ObjectName(attributes.get("canonicalName").asText());
- MBeanInfo info = server.getMBeanInfo(name);
- memoryUsageBroadcast.setName(name.toString());
-
- for (MBeanAttributeInfo attribute : Arrays.asList(info.getAttributes())) {
- switch (attribute.getName()) {
- case "Verbose":
- memoryUsageBroadcast.setVerbose((boolean) server.getAttribute(name, attribute.getName()));
- break;
- case "ObjectPendingFinalizationCount":
- memoryUsageBroadcast.setObjectPendingFinalizationCount(Long.parseLong(server.getAttribute(name, attribute.getName()).toString()));
- break;
- case "HeapMemoryUsage":
- memoryUsageBroadcast.setHeapMemoryUsage((Long) ((CompositeDataSupport)server.getAttribute(name, attribute.getName())).get("used"));
- break;
- case "NonHeapMemoryUsage":
- memoryUsageBroadcast.setNonHeapMemoryUsage((Long) ((CompositeDataSupport)server.getAttribute(name, attribute.getName())).get("used"));
- break;
- default:
- break;
- }
- }
-
- return memoryUsageBroadcast;
- } catch (Exception ex) {
- LOGGER.error("Exception trying to deserialize MemoryUsageDeserializer object: {}", ex);
- return null;
- }
- }
-}
diff --git a/streams-monitoring/src/main/java/org/apache/streams/jackson/StreamsTaskCounterDeserializer.java b/streams-monitoring/src/main/java/org/apache/streams/jackson/StreamsTaskCounterDeserializer.java
deleted file mode 100644
index e5f5dcb..0000000
--- a/streams-monitoring/src/main/java/org/apache/streams/jackson/StreamsTaskCounterDeserializer.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-import org.apache.streams.pojo.json.StreamsTaskCounterBroadcast;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import org.slf4j.Logger;
-
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.util.Arrays;
-import javax.management.MBeanAttributeInfo;
-import javax.management.MBeanInfo;
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
-
-/**
- * StreamsTaskCounterDeserializer: a JsonDeserializer for StreamsTaskCounterBroadcast.
- */
-public class StreamsTaskCounterDeserializer extends JsonDeserializer<StreamsTaskCounterBroadcast> {
-
- private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(StreamsTaskCounterDeserializer.class);
-
- public StreamsTaskCounterDeserializer() {
-
- }
-
- @Override
- public StreamsTaskCounterBroadcast deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
- try {
- MBeanServer server = ManagementFactory.getPlatformMBeanServer();
-
- StreamsTaskCounterBroadcast streamsTaskCounterBroadcast = new StreamsTaskCounterBroadcast();
- JsonNode attributes = jsonParser.getCodec().readTree(jsonParser);
-
- ObjectName name = new ObjectName(attributes.get("canonicalName").asText());
- MBeanInfo info = server.getMBeanInfo(name);
- streamsTaskCounterBroadcast.setName(name.toString());
-
- for (MBeanAttributeInfo attribute : Arrays.asList(info.getAttributes())) {
- try {
- switch (attribute.getName()) {
- case "ErrorRate":
- streamsTaskCounterBroadcast.setErrorRate((double) server.getAttribute(name, attribute.getName()));
- break;
- case "NumEmitted":
- streamsTaskCounterBroadcast.setNumEmitted((long) server.getAttribute(name, attribute.getName()));
- break;
- case "NumReceived":
- streamsTaskCounterBroadcast.setNumReceived((long) server.getAttribute(name, attribute.getName()));
- break;
- case "NumUnhandledErrors":
- streamsTaskCounterBroadcast.setNumUnhandledErrors((long) server.getAttribute(name, attribute.getName()));
- break;
- case "AvgTime":
- streamsTaskCounterBroadcast.setAvgTime((double) server.getAttribute(name, attribute.getName()));
- break;
- case "MaxTime":
- streamsTaskCounterBroadcast.setMaxTime((long) server.getAttribute(name, attribute.getName()));
- break;
- default:
- break;
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserialize StreamsTaskCounterBroadcast object: {}", ex);
- }
- }
-
- return streamsTaskCounterBroadcast;
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserialize StreamsTaskCounterBroadcast object: {}", ex);
- return null;
- }
- }
-}
diff --git a/streams-monitoring/src/main/java/org/apache/streams/jackson/ThroughputQueueDeserializer.java b/streams-monitoring/src/main/java/org/apache/streams/jackson/ThroughputQueueDeserializer.java
deleted file mode 100644
index 35dbcd5..0000000
--- a/streams-monitoring/src/main/java/org/apache/streams/jackson/ThroughputQueueDeserializer.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-import org.apache.streams.pojo.json.ThroughputQueueBroadcast;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import org.slf4j.Logger;
-
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.util.Arrays;
-import javax.management.MBeanAttributeInfo;
-import javax.management.MBeanInfo;
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
-
-public class ThroughputQueueDeserializer extends JsonDeserializer<ThroughputQueueBroadcast> {
-
- private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(ThroughputQueueDeserializer.class);
-
- public ThroughputQueueDeserializer() {
-
- }
-
- @Override
- public ThroughputQueueBroadcast deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
- try {
- MBeanServer server = ManagementFactory.getPlatformMBeanServer();
-
- ThroughputQueueBroadcast throughputQueueBroadcast = new ThroughputQueueBroadcast();
- JsonNode attributes = jsonParser.getCodec().readTree(jsonParser);
-
- ObjectName name = new ObjectName(attributes.get("canonicalName").asText());
- MBeanInfo info = server.getMBeanInfo(name);
- throughputQueueBroadcast.setName(name.toString());
-
- for (MBeanAttributeInfo attribute : Arrays.asList(info.getAttributes())) {
- try {
- switch (attribute.getName()) {
- case "CurrentSize":
- throughputQueueBroadcast.setCurrentSize((long) server.getAttribute(name, attribute.getName()));
- break;
- case "AvgWait":
- throughputQueueBroadcast.setAvgWait((double) server.getAttribute(name, attribute.getName()));
- break;
- case "MaxWait":
- throughputQueueBroadcast.setMaxWait((long) server.getAttribute(name, attribute.getName()));
- break;
- case "Removed":
- throughputQueueBroadcast.setRemoved((long) server.getAttribute(name, attribute.getName()));
- break;
- case "Added":
- throughputQueueBroadcast.setAdded((long) server.getAttribute(name, attribute.getName()));
- break;
- case "Throughput":
- throughputQueueBroadcast.setThroughput((double) server.getAttribute(name, attribute.getName()));
- break;
- default:
- break;
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while trying to deserialize ThroughputQueueBroadcast object: {}", ex);
- }
- }
-
- return throughputQueueBroadcast;
- } catch (Exception ex) {
- return null;
- }
- }
-}
\ No newline at end of file
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/persist/MessagePersister.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/persist/MessagePersister.java
deleted file mode 100644
index 667e9f6..0000000
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/persist/MessagePersister.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.monitoring.persist;
-
-import java.util.List;
-
-/**
- * Interface to define how we persist messages (JMX/monitoring related).
- */
-public interface MessagePersister {
-
- /**
- * Given a list of messages, persist them out through whatever appropriate
- * broadcast mechanism (HTTP request, SLF4J log, etc.).
- * @param messages List of String messages
- * @return statusCode represents whether or not the persist was successful
- */
- int persistMessages(List<String> messages);
-}
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/persist/impl/BroadcastMessagePersister.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/persist/impl/BroadcastMessagePersister.java
deleted file mode 100644
index cf95f7e..0000000
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/persist/impl/BroadcastMessagePersister.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.monitoring.persist.impl;
-
-import org.apache.streams.monitoring.persist.MessagePersister;
-
-import org.apache.http.HttpResponse;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.entity.UrlEncodedFormEntity;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.message.BasicNameValuePair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class BroadcastMessagePersister implements MessagePersister {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(BroadcastMessagePersister.class);
- private String broadcastUri;
-
- public BroadcastMessagePersister(String broadcastUri) {
- this.broadcastUri = broadcastUri;
- }
-
- @Override
- /*
- * Given a list of messages as Strings, broadcast them to the broadcastUri
- * (if one is defined)
- * @param messages
- * @return int status code from POST response
- */
- public int persistMessages(List<String> messages) {
- int responseCode = -1;
-
- if (broadcastUri != null) {
- try {
- HttpClient client = HttpClients.createDefault();
- HttpPost post = new HttpPost(broadcastUri);
-
- post.setHeader("User-Agent", "Streams");
-
- List<NameValuePair> urlParameters = new ArrayList<>();
- urlParameters.add(new BasicNameValuePair("messages", serializeMessages(messages)));
-
- post.setEntity(new UrlEncodedFormEntity(urlParameters, "UTF-8"));
-
- HttpResponse response = client.execute(post);
- responseCode = response.getStatusLine().getStatusCode();
-
- LOGGER.debug("Broadcast {} messages to URI: {}", messages.size(), broadcastUri);
- } catch (Exception ex) {
- LOGGER.error("Failed to broadcast message to URI: {}, exception: {}", broadcastUri, ex);
- }
- }
-
- return responseCode;
- }
-
- /**
- * Given a List of String messages, convert them to a JSON array.
- * @param messages List of String messages
- * @return Serialized version of this JSON array
- */
- private String serializeMessages(List<String> messages) {
- String ser = "{\"messages\":[";
-
- for (String message : messages) {
- if (messages.get(messages.size() - 1).equals(message)) {
- ser += message + "]}";
- } else {
- ser += message + ",";
- }
- }
-
- return ser;
- }
-}
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/persist/impl/LogstashUdpMessagePersister.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/persist/impl/LogstashUdpMessagePersister.java
deleted file mode 100644
index c697661..0000000
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/persist/impl/LogstashUdpMessagePersister.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.monitoring.persist.impl;
-
-import org.apache.streams.monitoring.persist.MessagePersister;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.DatagramPacket;
-import java.net.DatagramSocket;
-import java.net.InetSocketAddress;
-import java.net.SocketException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-public class LogstashUdpMessagePersister implements MessagePersister {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(LogstashUdpMessagePersister.class);
- private String broadcastUri;
- URI uri;
-
- public LogstashUdpMessagePersister(String broadcastUri) {
- this.broadcastUri = broadcastUri;
- setup();
- }
-
- /**
- * setup.
- */
- public void setup() {
-
- try {
- uri = new URI(broadcastUri);
- } catch (URISyntaxException ex) {
- LOGGER.error(ex.getMessage());
- }
-
- }
-
- @Override
- /**
- * Given a list of messages as Strings, broadcast them to the broadcastUri
- * (if one is defined)
- * @param messages
- * @return int status code from POST response
- */
- public int persistMessages(List<String> messages) {
- int responseCode = -1;
-
- if (broadcastUri != null) {
- DatagramSocket socket = null;
- try {
- socket = new DatagramSocket();
- } catch (SocketException ex) {
- LOGGER.error("Metrics Broadcast Setup Failed: " + ex.getMessage());
- }
- try {
- ByteBuffer toWrite = ByteBuffer.wrap(serializeMessages(messages).getBytes());
- byte[] byteArray = toWrite.array();
- DatagramPacket packet = new DatagramPacket(byteArray, byteArray.length);
- socket.connect(new InetSocketAddress(uri.getHost(), uri.getPort()));
- socket.send(packet);
- } catch ( Exception ex ) {
- LOGGER.error("Metrics Broadcast Failed: " + ex.getMessage());
- } finally {
- socket.close();
- }
- }
-
- return responseCode;
- }
-
- /**
- * Given a List of String messages, convert them to a JSON array.
- * @param messages List of String messages
- * @return Serialized version of this JSON array
- */
- private String serializeMessages(List<String> messages) {
-
- StringBuilder jsonLines = new StringBuilder();
- for (String message : messages) {
- jsonLines.append(message).append('\n');
- }
-
- return jsonLines.toString();
- }
-
-}
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/persist/impl/Slf4jMessagePersister.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/persist/impl/Slf4jMessagePersister.java
deleted file mode 100644
index b237871..0000000
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/persist/impl/Slf4jMessagePersister.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.monitoring.persist.impl;
-
-import org.apache.streams.monitoring.persist.MessagePersister;
-
-import org.slf4j.Logger;
-
-import java.util.List;
-
-/**
- * Persist montoring messages to SLF4J.
- */
-public class Slf4jMessagePersister implements MessagePersister {
-
- private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(Slf4jMessagePersister.class);
- private static final int SUCCESS_STATUS = 0;
- private static final int FAILURE_STATUS = -1;
-
- public Slf4jMessagePersister() {
-
- }
-
- @Override
- public int persistMessages(List<String> messages) {
-
- for (String message : messages) {
- LOGGER.info(message);
- }
-
- return SUCCESS_STATUS;
- }
-}
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
deleted file mode 100644
index 5b85e68..0000000
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.monitoring.tasks;
-
-import org.apache.streams.config.ComponentConfigurator;
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.jackson.DatumStatusCounterDeserializer;
-import org.apache.streams.jackson.MemoryUsageDeserializer;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.jackson.StreamsTaskCounterDeserializer;
-import org.apache.streams.jackson.ThroughputQueueDeserializer;
-import org.apache.streams.local.monitoring.MonitoringConfiguration;
-import org.apache.streams.monitoring.persist.MessagePersister;
-import org.apache.streams.monitoring.persist.impl.BroadcastMessagePersister;
-import org.apache.streams.monitoring.persist.impl.LogstashUdpMessagePersister;
-import org.apache.streams.monitoring.persist.impl.Slf4jMessagePersister;
-import org.apache.streams.pojo.json.Broadcast;
-import org.apache.streams.pojo.json.DatumStatusCounterBroadcast;
-import org.apache.streams.pojo.json.MemoryUsageBroadcast;
-import org.apache.streams.pojo.json.StreamsTaskCounterBroadcast;
-import org.apache.streams.pojo.json.ThroughputQueueBroadcast;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import org.slf4j.Logger;
-
-import java.lang.management.ManagementFactory;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import javax.management.MBeanServer;
-import javax.management.NotificationBroadcasterSupport;
-import javax.management.ObjectName;
-
-/**
- * This thread runs inside of a Streams runtime and periodically persists information
- * from relevant JMX beans.
- */
-public class BroadcastMonitorThread extends NotificationBroadcasterSupport implements Runnable {
-
- private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(BroadcastMonitorThread.class);
- private static MBeanServer server;
-
- private MonitoringConfiguration configuration;
- private URI broadcastUri = null;
- private MessagePersister messagePersister;
- private volatile boolean keepRunning;
-
- private static ObjectMapper objectMapper = StreamsJacksonMapper.getInstance();
-
- /**
- * DEPRECATED
- * Please initialize logging with monitoring object via typesafe.
- * @param streamConfig streamConfig map.
- */
- @Deprecated
- public BroadcastMonitorThread(Map<String, Object> streamConfig) {
- this(objectMapper.convertValue(streamConfig, MonitoringConfiguration.class));
- }
-
- public BroadcastMonitorThread(StreamsConfiguration streamConfig) {
- this(objectMapper.convertValue(streamConfig.getAdditionalProperties().get("monitoring"), MonitoringConfiguration.class));
- }
-
- /**
- * BroadcastMonitorThread constructor - uses supplied MonitoringConfiguration.
- * @param configuration MonitoringConfiguration
- */
- public BroadcastMonitorThread(MonitoringConfiguration configuration) {
-
- this.configuration = configuration;
- if ( this.configuration == null ) {
- this.configuration = new ComponentConfigurator<>(MonitoringConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig().atPath("monitoring"));
- }
-
- LOGGER.info("BroadcastMonitorThread created");
-
- initializeObjectMapper();
-
- prepare();
-
- LOGGER.info("BroadcastMonitorThread initialized");
-
- }
-
- /**
- * Initialize our object mapper with all of our bean's custom deserializers.
- * This way we can convert them to and from Strings dictated by our
- * POJOs which are generated from JSON schemas.
- */
- private void initializeObjectMapper() {
- SimpleModule simpleModule = new SimpleModule();
-
- simpleModule.addDeserializer(MemoryUsageBroadcast.class, new MemoryUsageDeserializer());
- simpleModule.addDeserializer(ThroughputQueueBroadcast.class, new ThroughputQueueDeserializer());
- simpleModule.addDeserializer(StreamsTaskCounterBroadcast.class, new StreamsTaskCounterDeserializer());
- simpleModule.addDeserializer(DatumStatusCounterBroadcast.class, new DatumStatusCounterDeserializer());
-
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- /**
- * Get all relevant JMX beans, convert their values to strings, and then persist them.
- */
- @Override
- public void run() {
- LOGGER.info("BroadcastMonitorThread running");
- while (keepRunning) {
- try {
- List<String> messages = new ArrayList<>();
- Set<ObjectName> beans = server.queryNames(null, null);
-
- for (ObjectName name : beans) {
- String item = objectMapper.writeValueAsString(name);
- Broadcast broadcast = null;
-
- if (name.getKeyPropertyList().get("type") != null) {
- if (name.getKeyPropertyList().get("type").equals("ThroughputQueue")) {
- broadcast = objectMapper.readValue(item, ThroughputQueueBroadcast.class);
- } else if (name.getKeyPropertyList().get("type").equals("StreamsTaskCounter")) {
- broadcast = objectMapper.readValue(item, StreamsTaskCounterBroadcast.class);
- } else if (name.getKeyPropertyList().get("type").equals("DatumStatusCounter")) {
- broadcast = objectMapper.readValue(item, DatumStatusCounterBroadcast.class);
- } else if (name.getKeyPropertyList().get("type").equals("Memory")) {
- broadcast = objectMapper.readValue(item, MemoryUsageBroadcast.class);
- }
-
- if (broadcast != null) {
- messages.add(objectMapper.writeValueAsString(broadcast));
- }
- }
- }
-
- messagePersister.persistMessages(messages);
- Thread.sleep(configuration.getMonitoringBroadcastIntervalMs());
- } catch (InterruptedException ex) {
- LOGGER.debug("Broadcast Monitor Interrupted!");
- Thread.currentThread().interrupt();
- this.keepRunning = false;
- } catch (Exception ex) {
- LOGGER.error("Exception: {}", ex);
- this.keepRunning = false;
- }
- }
- }
-
- /**
- * prepare for execution.
- */
- public void prepare() {
-
- keepRunning = true;
-
- LOGGER.info("BroadcastMonitorThread setup " + this.configuration);
-
- server = ManagementFactory.getPlatformMBeanServer();
-
- if (this.configuration != null && this.configuration.getBroadcastURI() != null) {
-
- try {
- broadcastUri = new URI(configuration.getBroadcastURI());
- } catch (Exception ex) {
- LOGGER.error("invalid URI: ", ex);
- }
-
- if (broadcastUri != null) {
- switch (broadcastUri.getScheme()) {
- case "http":
- messagePersister = new BroadcastMessagePersister(broadcastUri.toString());
- break;
- case "udp":
- messagePersister = new LogstashUdpMessagePersister(broadcastUri.toString());
- break;
- default:
- LOGGER.error("You need to specify a broadcast URI with either a HTTP or UDP protocol defined.");
- throw new RuntimeException();
- }
- } else {
- messagePersister = new Slf4jMessagePersister();
- }
- } else {
- messagePersister = new Slf4jMessagePersister();
- }
-
- }
-
- public void shutdown() {
- this.keepRunning = false;
- LOGGER.debug("Shutting down BroadcastMonitor Thread");
- }
-
- public String getBroadcastUri() {
- return configuration.getBroadcastURI();
- }
-
- public long getWaitTime() {
- return configuration.getMonitoringBroadcastIntervalMs();
- }
-
-}
diff --git a/streams-monitoring/src/main/jsonschema/org/apache/streams/local/monitoring/MonitoringConfiguration.json b/streams-monitoring/src/main/jsonschema/org/apache/streams/local/monitoring/MonitoringConfiguration.json
deleted file mode 100644
index 2c50cef..0000000
--- a/streams-monitoring/src/main/jsonschema/org/apache/streams/local/monitoring/MonitoringConfiguration.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "title": "object",
- "javaType": "org.apache.streams.local.monitoring.MonitoringConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "Configuration for local runtime monitoring",
- "properties": {
- "broadcastURI": {
- "type": "string",
- "description": "URI for monitoring statistics"
- },
- "monitoring_broadcast_interval_ms": {
- "type": "integer",
- "description": "Milliseconds between publish events",
- "default": 30000
- }
- }
-}
\ No newline at end of file
diff --git a/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/Broadcast.json b/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/Broadcast.json
deleted file mode 100644
index 6bd7ad6..0000000
--- a/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/Broadcast.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "title": "object",
- "javaType": "org.apache.streams.pojo.json.Broadcast",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "Base Broadcast class",
- "properties": {
- "name": {
- "type": "string",
- "description": "Name of the MBean"
- },
- "streamIdentifier": {
- "type": "string",
- "description": "The name of the Stream that is currently executing"
- },
- "startedAt": {
- "type": "integer",
- "description": "Milliseconds since epoch when this Stream was started"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/DatumStatusCounterBroadcast.json b/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/DatumStatusCounterBroadcast.json
deleted file mode 100644
index 21d9f6a..0000000
--- a/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/DatumStatusCounterBroadcast.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "title": "object",
- "extends": {
- "$ref": "./Broadcast.json"
- },
- "javaType": "org.apache.streams.pojo.json.DatumStatusCounterBroadcast",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "Snapshot of the DatumStatusCounter",
- "properties": {
- "passed": {
- "type": "boolean",
- "description": "Number of objects that have passed"
- },
- "failed": {
- "type": "boolean",
- "description": "Number of objects that have faile"
- },
- "name": {
- "type": "string",
- "description": "Name of the MBean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/MemoryUsageBroadcast.json b/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/MemoryUsageBroadcast.json
deleted file mode 100644
index b29284e..0000000
--- a/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/MemoryUsageBroadcast.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "title": "object",
- "extends": {
- "$ref": "./Broadcast.json"
- },
- "javaType": "org.apache.streams.pojo.json.MemoryUsageBroadcast",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "Snapshot of current memory usage",
- "properties": {
- "verbose": {
- "type": "boolean",
- "description": "Whether or not this is verbose"
- },
- "objectPendingFinalizationCount": {
- "type": "integer",
- "description": "The number of objects that are pending finalization"
- },
- "heapMemoryUsage": {
- "type": "integer",
- "description": "The amount of heap memory we are currently using"
- },
- "nonHeapMemoryUsage": {
- "type": "integer",
- "description": "The amount of non-heap memory we are using"
- },
- "name": {
- "type": "string",
- "description": "The name of this MBean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/StreamsTaskCounterBroadcast.json b/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/StreamsTaskCounterBroadcast.json
deleted file mode 100644
index bfb7817..0000000
--- a/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/StreamsTaskCounterBroadcast.json
+++ /dev/null
@@ -1,45 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "title": "object",
- "javaType": "org.apache.streams.pojo.json.StreamsTaskCounterBroadcast",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {
- "$ref": "./Broadcast.json"
- },
- "description": "Snapshot of a Stream Task Counter",
- "properties": {
- "errorRate": {
- "type": "double",
- "description": "Rate of failed items"
- },
- "numEmitted": {
- "type": "integer",
- "description": "Number of items that have been emitted"
- },
- "numReceived": {
- "type": "integer",
- "description": "Number of items this Task has received"
- },
- "numUnhandledErrors": {
- "type": "integer",
- "description": "Number of unhandled errors"
- },
- "avgTime": {
- "type": "double",
- "description": "Average amount of time an item spent in this Task"
- },
- "maxTime": {
- "type": "integer",
- "description": "Longest amount of time an item spent in this Task"
- },
- "name": {
- "type": "string",
- "description": "Name of the MBean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/ThroughputQueueBroadcast.json b/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/ThroughputQueueBroadcast.json
deleted file mode 100644
index 14f949d..0000000
--- a/streams-monitoring/src/main/jsonschema/org/apache/streams/pojo/json/ThroughputQueueBroadcast.json
+++ /dev/null
@@ -1,45 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "title": "object",
- "extends": {
- "$ref": "./Broadcast.json"
- },
- "javaType": "org.apache.streams.pojo.json.ThroughputQueueBroadcast",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "Snapshot of a ThroughputQueue's performance",
- "properties": {
- "currentSize": {
- "type": "integer",
- "description": "Current size of the queue"
- },
- "avgWait": {
- "type": "double",
- "description": "Average wait time"
- },
- "maxWait": {
- "type": "double",
- "description": "Maximum wait time"
- },
- "removed": {
- "type": "integer",
- "description": "Number of elements removed from the queue"
- },
- "added": {
- "type": "integer",
- "description": "Number of elements added to the queue"
- },
- "throughput": {
- "type": "double",
- "description": "Number of elements that have passed through the queue per second"
- },
- "name": {
- "type": "string",
- "description": "Name of the MBean"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-monitoring/src/main/resources/reference.conf b/streams-monitoring/src/main/resources/reference.conf
deleted file mode 100644
index ef7834f..0000000
--- a/streams-monitoring/src/main/resources/reference.conf
+++ /dev/null
@@ -1,3 +0,0 @@
-monitoring {
- monitoring_broadcast_interval_ms = 30000
-}
\ No newline at end of file
diff --git a/streams-monitoring/src/site/markdown/index.md b/streams-monitoring/src/site/markdown/index.md
deleted file mode 100644
index 1ffab81..0000000
--- a/streams-monitoring/src/site/markdown/index.md
+++ /dev/null
@@ -1,8 +0,0 @@
-streams-monitoring
-==============
-
-streams-monitoring contains classes and interfaces for monitoring executing streams.
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-monitoring/src/test/java/org/apache/streams/jackson/MemoryUsageDeserializerTest.java b/streams-monitoring/src/test/java/org/apache/streams/jackson/MemoryUsageDeserializerTest.java
deleted file mode 100644
index 8bf3219..0000000
--- a/streams-monitoring/src/test/java/org/apache/streams/jackson/MemoryUsageDeserializerTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-import org.apache.streams.pojo.json.MemoryUsageBroadcast;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import org.apache.commons.lang3.StringUtils;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-import static org.junit.Assert.assertNotNull;
-
-public class MemoryUsageDeserializerTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MemoryUsageDeserializerTest.class);
- private ObjectMapper objectMapper;
-
- /**
- * setup.
- */
- @Before
- public void setup() {
- objectMapper = StreamsJacksonMapper.getInstance();
- SimpleModule simpleModule = new SimpleModule();
- simpleModule.addDeserializer(MemoryUsageBroadcast.class, new MemoryUsageDeserializer());
- objectMapper.registerModule(simpleModule);
- objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- @Test
- public void serDeTest() {
- InputStream is = MemoryUsageDeserializerTest.class.getResourceAsStream("/MemoryUsageObjects.json");
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- try {
- while (br.ready()) {
- String line = br.readLine();
- if (!StringUtils.isEmpty(line)) {
- LOGGER.info("raw: {}", line);
- MemoryUsageBroadcast broadcast = objectMapper.readValue(line, MemoryUsageBroadcast.class);
-
- LOGGER.info("activity: {}", broadcast);
-
- assertNotNull(broadcast);
- assertNotNull(broadcast.getVerbose());
- assertNotNull(broadcast.getObjectPendingFinalizationCount());
- assertNotNull(broadcast.getHeapMemoryUsage());
- assertNotNull(broadcast.getNonHeapMemoryUsage());
- assertNotNull(broadcast.getName());
- }
- }
- } catch (Exception ex) {
- LOGGER.error("Exception while testing serializability: {}", ex);
- }
- }
-}
diff --git a/streams-monitoring/src/test/java/org/apache/streams/monitoring/persist/impl/BroadcastMessagePersisterTest.java b/streams-monitoring/src/test/java/org/apache/streams/monitoring/persist/impl/BroadcastMessagePersisterTest.java
deleted file mode 100644
index 7cc43d5..0000000
--- a/streams-monitoring/src/test/java/org/apache/streams/monitoring/persist/impl/BroadcastMessagePersisterTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.monitoring.persist.impl;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-
-public class BroadcastMessagePersisterTest {
-
- @Test
- public void testFailedPersist() {
- BroadcastMessagePersister persister = new BroadcastMessagePersister("http://fake.url.com/fake_endpointasdfasdfas");
-
- List<String> messages = new ArrayList<>();
- for (int x = 0; x < 10; x++) {
- messages.add("Fake_message #" + x);
- }
-
- int statusCode = persister.persistMessages(messages);
-
- assertNotNull(statusCode);
- assertNotEquals(statusCode, 200);
- }
-
- @Test
- public void testInvalidUrl() {
- BroadcastMessagePersister persister = new BroadcastMessagePersister("h");
-
- List<String> messages = new ArrayList<>();
- for (int x = 0; x < 10; x++) {
- messages.add("Fake_message #" + x);
- }
-
- int statusCode = persister.persistMessages(messages);
-
- assertNotNull(statusCode);
- assertEquals(statusCode, -1);
- }
-}
diff --git a/streams-monitoring/src/test/java/org/apache/streams/monitoring/persist/impl/LogstashUdpMessagePersisterTest.java b/streams-monitoring/src/test/java/org/apache/streams/monitoring/persist/impl/LogstashUdpMessagePersisterTest.java
deleted file mode 100644
index 26076d9..0000000
--- a/streams-monitoring/src/test/java/org/apache/streams/monitoring/persist/impl/LogstashUdpMessagePersisterTest.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.monitoring.persist.impl;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-
-import java.io.IOException;
-import java.net.DatagramPacket;
-import java.net.DatagramSocket;
-import java.net.SocketException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public class LogstashUdpMessagePersisterTest {
-
- private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(LogstashUdpMessagePersisterTest.class);
-
- private DatagramSocket socket = null;
-
- /**
- * setup.
- */
- @Before
- public void setup() {
- try {
- socket = new DatagramSocket(56789);
- } catch (SocketException ex) {
- LOGGER.error("Metrics Broadcast Test Setup Failed: " + ex.getMessage());
- }
- }
-
-
- @Test
- public void testFailedPersist() {
- LogstashUdpMessagePersister persister = new LogstashUdpMessagePersister("udp://127.0.0.1:56789");
-
- List<String> messageArray = new ArrayList<>();
- for (int x = 0; x < 10; x ++) {
- messageArray.add("Fake_message #" + x);
- }
-
- persister.persistMessages(messageArray);
- byte[] receiveData = new byte[1024];
-
- DatagramPacket messageDatagram = new DatagramPacket(receiveData, receiveData.length);
-
- try {
- socket.receive(messageDatagram);
- Assert.assertNotNull(messageDatagram);
- List<String> messages = Stream.of((new String(messageDatagram.getData())).split("\n")).collect(Collectors.toList());
- Assert.assertEquals(messageArray, messages.subList(0,10));
- } catch (IOException ex) {
- LOGGER.error("Metrics Broadcast Test Failed: " + ex.getMessage());
- }
-
- }
-
-}
diff --git a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
deleted file mode 100644
index ba0952f..0000000
--- a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.monitoring.tasks;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.local.monitoring.MonitoringConfiguration;
-
-import org.junit.Test;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-public class BroadcastMonitorThreadTest {
-
- private ExecutorService executor;
-
- @Test
- public void testThreadEmptyBeanConfig() {
- StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration();
- BroadcastMonitorThread thread = new BroadcastMonitorThread(streamsConfiguration);
- testThread(thread);
- }
-
-
-
-
- @Test
- public void testThreadStreamsConfig() {
-
- StreamsConfiguration streams = new StreamsConfiguration();
- MonitoringConfiguration monitoring = new MonitoringConfiguration();
- monitoring.setBroadcastURI("http://fakeurl.com/fake");
- monitoring.setMonitoringBroadcastIntervalMs(30000L);
- streams.setAdditionalProperty("monitoring", monitoring);
- BroadcastMonitorThread thread = new BroadcastMonitorThread(streams);
- testThread(thread);
- }
-
- /**
- * Base Test.
- * @param thread BroadcastMonitorThread
- */
- public void testThread(BroadcastMonitorThread thread) {
- long testRunLength = thread.getWaitTime() * 1;
- executor = Executors.newFixedThreadPool(1);
- executor.submit(thread);
-
- try {
- Thread.sleep(testRunLength);
- } catch (InterruptedException ex) {
- Thread.currentThread().interrupt();
- }
-
- executor.shutdown();
- }
-
-}
diff --git a/streams-monitoring/src/test/resources/MemoryUsageObjects.json b/streams-monitoring/src/test/resources/MemoryUsageObjects.json
deleted file mode 100644
index f955fdc..0000000
--- a/streams-monitoring/src/test/resources/MemoryUsageObjects.json
+++ /dev/null
@@ -1 +0,0 @@
-{"canonicalName":"java.lang:type=Memory","pattern":false,"domainPattern":false,"propertyPattern":false,"propertyListPattern":false,"propertyValuePattern":false,"domain":"java.lang","keyPropertyList":{"type":"Memory"},"keyPropertyListString":"type=Memory","canonicalKeyPropertyListString":"type=Memory"}
\ No newline at end of file
diff --git a/streams-plugins/pom.xml b/streams-plugins/pom.xml
deleted file mode 100644
index e4b3e89..0000000
--- a/streams-plugins/pom.xml
+++ /dev/null
@@ -1,150 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>apache-streams</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugins</artifactId>
-
- <packaging>pom</packaging>
- <name>streams-plugins</name>
-
- <properties>
- <maven-plugin.version>3.3.3</maven-plugin.version>
- <maven-plugin-plugin.version>3.4</maven-plugin-plugin.version>
- <maven-plugin-tools.version>3.3</maven-plugin-tools.version>
- <maven-plugin-testing.version>3.3.0</maven-plugin-testing.version>
- <maven-verifier.version>1.6</maven-verifier.version>
- </properties>
-
- <modules>
- <module>streams-plugin-cassandra</module>
- <module>streams-plugin-elasticsearch</module>
- <module>streams-plugin-hbase</module>
- <module>streams-plugin-hive</module>
- <module>streams-plugin-pig</module>
- <module>streams-plugin-pojo</module>
- <module>streams-plugin-scala</module>
- </modules>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-aether-provider</artifactId>
- <version>${maven-plugin.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- <version>${resources.plugin.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.maven</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-artifact</artifactId>
- <version>${maven-plugin.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-compat</artifactId>
- <version>${maven-plugin.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.shared</groupId>
- <artifactId>maven-verifier</artifactId>
- <version>${maven-verifier.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-core</artifactId>
- <version>${maven-plugin.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.sonatype.aether</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-plugin-plugin</artifactId>
- <version>${maven-plugin-plugin.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>commons-logging</artifactId>
- <groupId>commons-logging</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-plugin-api</artifactId>
- <version>${maven-plugin.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-api</artifactId>
- <version>${maven-plugin-tools.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-annotations</artifactId>
- <version>${maven-plugin-tools.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-generators</artifactId>
- <version>${maven-plugin-tools.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-testing</groupId>
- <artifactId>maven-plugin-testing-harness</artifactId>
- <version>${maven-plugin-testing.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
-</project>
diff --git a/streams-plugins/streams-plugin-cassandra/pom.xml b/streams-plugins/streams-plugin-cassandra/pom.xml
deleted file mode 100644
index 926240f..0000000
--- a/streams-plugins/streams-plugin-cassandra/pom.xml
+++ /dev/null
@@ -1,196 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
- <artifactId>streams-plugin-cassandra</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <packaging>maven-plugin</packaging>
-
- <parent>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugins</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <properties>
- <cassandra.version>3.11.0</cassandra.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>commons-logging</artifactId>
- <groupId>commons-logging</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-aether-provider</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-artifact</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-compat</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-plugin-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.shared</groupId>
- <artifactId>maven-verifier</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.codehaus.plexus</groupId>
- <artifactId>plexus-utils</artifactId>
- <version>3.1.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-plugin-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-generators</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-testing</groupId>
- <artifactId>maven-plugin-testing-harness</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>org.joda</groupId>
- <artifactId>joda-convert</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <artifactId>maven-plugin-plugin</artifactId>
- <version>3.4</version>
- <configuration>
- <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
- </configuration>
- <executions>
- <execution>
- <id>mojo-descriptor</id>
- <phase>process-classes</phase>
- <goals>
- <goal>descriptor</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>resource-dependencies</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includes>**/*.json</includes>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
-</project>
diff --git a/streams-plugins/streams-plugin-cassandra/src/main/java/org/apache/streams/plugins/cassandra/StreamsCassandraGenerationConfig.java b/streams-plugins/streams-plugin-cassandra/src/main/java/org/apache/streams/plugins/cassandra/StreamsCassandraGenerationConfig.java
deleted file mode 100644
index 971b99f..0000000
--- a/streams-plugins/streams-plugin-cassandra/src/main/java/org/apache/streams/plugins/cassandra/StreamsCassandraGenerationConfig.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.cassandra;
-
-import org.apache.streams.util.schema.GenerationConfig;
-
-import org.jsonschema2pojo.DefaultGenerationConfig;
-import org.jsonschema2pojo.util.URLUtil;
-
-import java.io.File;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Configures StreamsCassandraResourceGenerator.
- */
-public class StreamsCassandraGenerationConfig extends DefaultGenerationConfig implements GenerationConfig {
-
- public String getSourceDirectory() {
- return sourceDirectory;
- }
-
- public List<String> getSourcePaths() {
- return sourcePaths;
- }
-
- private String sourceDirectory;
- private List<String> sourcePaths = new ArrayList<String>();
- private String targetDirectory;
- private int maxDepth = 1;
-
- public Set<String> getExclusions() {
- return exclusions;
- }
-
- public void setExclusions(Set<String> exclusions) {
- this.exclusions = exclusions;
- }
-
- private Set<String> exclusions = new HashSet<String>();
-
- public int getMaxDepth() {
- return maxDepth;
- }
-
- public void setSourceDirectory(String sourceDirectory) {
- this.sourceDirectory = sourceDirectory;
- }
-
- public void setSourcePaths(List<String> sourcePaths) {
- this.sourcePaths = sourcePaths;
- }
-
- public void setTargetDirectory(String targetDirectory) {
- this.targetDirectory = targetDirectory;
- }
-
- public File getTargetDirectory() {
- return new File(targetDirectory);
- }
-
- /**
- * get all sources.
- * @return Iterator of URL
- */
- public Iterator<URL> getSource() {
- if (null != sourceDirectory) {
- return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator();
- }
- List<URL> sourceUrls = new ArrayList<URL>();
- if ( sourcePaths != null && sourcePaths.size() > 0) {
- for (String source : sourcePaths) {
- sourceUrls.add(URLUtil.parseURL(source));
- }
- }
- return sourceUrls.iterator();
- }
-
- public void setMaxDepth(int maxDepth) {
- this.maxDepth = maxDepth;
- }
-}
diff --git a/streams-plugins/streams-plugin-cassandra/src/main/java/org/apache/streams/plugins/cassandra/StreamsCassandraResourceGenerator.java b/streams-plugins/streams-plugin-cassandra/src/main/java/org/apache/streams/plugins/cassandra/StreamsCassandraResourceGenerator.java
deleted file mode 100644
index 7c0df9d..0000000
--- a/streams-plugins/streams-plugin-cassandra/src/main/java/org/apache/streams/plugins/cassandra/StreamsCassandraResourceGenerator.java
+++ /dev/null
@@ -1,409 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.cassandra;
-
-import org.apache.streams.util.schema.FieldType;
-import org.apache.streams.util.schema.FieldUtil;
-import org.apache.streams.util.schema.Schema;
-import org.apache.streams.util.schema.SchemaStore;
-import org.apache.streams.util.schema.SchemaStoreImpl;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang3.StringUtils;
-import org.jsonschema2pojo.util.URLUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.net.URI;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-
-import static org.apache.streams.util.schema.FileUtil.dropExtension;
-import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
-import static org.apache.streams.util.schema.FileUtil.resolveRecursive;
-import static org.apache.streams.util.schema.FileUtil.writeFile;
-
-/**
- * Resource Generator for Cassandra.
- */
-public class StreamsCassandraResourceGenerator implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsCassandraResourceGenerator.class);
-
- private static final String LS = System.getProperty("line.separator");
-
- private StreamsCassandraGenerationConfig config;
-
- private SchemaStore schemaStore = new SchemaStoreImpl();
-
- private int currentDepth = 0;
-
- /**
- * Run from CLI without Maven
- *
- * <p/>
- * java -jar streams-plugin-cassandra-jar-with-dependencies.jar StreamsCassandraResourceGenerator src/main/jsonschema target/generated-resources
- *
- * @param args [sourceDirectory, targetDirectory]
- */
- public static void main(String[] args) {
- StreamsCassandraGenerationConfig config = new StreamsCassandraGenerationConfig();
-
- String sourceDirectory = "./src/main/jsonschema";
- String targetDirectory = "./target/generated-resources/cassandra";
-
- if ( args.length > 0 ) {
- sourceDirectory = args[0];
- }
- if ( args.length > 1 ) {
- targetDirectory = args[1];
- }
-
- config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory(targetDirectory);
-
- StreamsCassandraResourceGenerator streamsCassandraResourceGenerator = new StreamsCassandraResourceGenerator(config);
- streamsCassandraResourceGenerator.run();
- }
-
- public StreamsCassandraResourceGenerator(StreamsCassandraGenerationConfig config) {
- this.config = config;
- }
-
- @Override
- public void run() {
-
- Objects.requireNonNull(config);
-
- generate(config);
-
- }
-
- /**
- * run generate using supplied StreamsCassandraGenerationConfig.
- * @param config StreamsCassandraGenerationConfig
- */
- public void generate(StreamsCassandraGenerationConfig config) {
-
- LinkedList<File> sourceFiles = new LinkedList<>();
-
- for (Iterator<URL> sources = config.getSource(); sources.hasNext();) {
- URL source = sources.next();
- sourceFiles.add(URLUtil.getFileFromURL(source));
- }
-
- LOGGER.info("Seeded with {} source paths:", sourceFiles.size());
-
- resolveRecursive(config, sourceFiles);
-
- LOGGER.info("Resolved {} schema files:", sourceFiles.size());
-
- for (File item : sourceFiles) {
- schemaStore.create(item.toURI());
- }
-
- LOGGER.info("Identified {} objects:", schemaStore.getSize());
-
- String outputFile = config.getTargetDirectory() + "/" + "types.cql";
- StringBuilder typesContent = new StringBuilder();
-
- for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) {
- Schema schema = schemaIterator.next();
- currentDepth = 0;
- if ( schema.getUri().getScheme().equals("file")) {
- String inputFile = schema.getUri().getPath();
- String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
- for (String sourcePath : config.getSourcePaths()) {
- resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
- }
-
- String resourceId = schemaSymbol(schema);
-
- LOGGER.info("Processing {}", resourcePath);
-
- String resourceContent = generateResource(schema, resourceId);
-
- typesContent.append(resourceContent);
-
- LOGGER.info("Added {}", resourceId);
- }
- }
-
- writeFile(outputFile, typesContent.toString());
-
- }
-
- /**
- * generateResource String from schema and resourceId.
- * @param schema Schema
- * @param resourceId String
- * @return CREATE TYPE ...
- */
- public String generateResource(Schema schema, String resourceId) {
- StringBuilder resourceBuilder = new StringBuilder();
- resourceBuilder.append("CREATE TYPE ");
- resourceBuilder.append(resourceId);
- resourceBuilder.append(" IF NOT EXISTS (");
- resourceBuilder.append(LS);
- resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId, ' ');
- resourceBuilder.append(");");
- resourceBuilder.append(LS);
- return resourceBuilder.toString();
- }
-
- protected StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId, Character seperator) {
- ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId);
- if ( propertiesNode.get("id") != null ) {
- builder.append("id text PRIMARY KEY,");
- builder.append(LS);
- propertiesNode.remove("id");
- }
- if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) {
- builder = appendPropertiesNode(builder, schema, propertiesNode, seperator);
- }
- return builder;
- }
-
- private StringBuilder appendValueField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- builder.append(cqlEscape(fieldId));
- builder.append(seperator);
- builder.append(cqlType(fieldType));
- return builder;
- }
-
- protected StringBuilder appendArrayItems(StringBuilder builder, Schema schema, String fieldId, ObjectNode itemsNode, Character seperator) {
- // not safe to append nothing
- Objects.requireNonNull(builder);
- if ( itemsNode == null ) {
- return builder;
- }
- if ( itemsNode.has("type")) {
- try {
- FieldType itemType = FieldUtil.determineFieldType(itemsNode);
- switch ( itemType ) {
- case OBJECT:
- Schema objectSchema = null;
- URI parentUri = null;
- if ( itemsNode.has("$ref") || itemsNode.has("extends") ) {
- JsonNode refNode = itemsNode.get("$ref");
- JsonNode extendsNode = itemsNode.get("extends");
- if (refNode != null && refNode.isValueNode()) {
- parentUri = URI.create(refNode.asText());
- } else if (extendsNode != null && extendsNode.isObject()) {
- parentUri = URI.create(extendsNode.get("$ref").asText());
- }
- URI absoluteUri;
- if (parentUri.isAbsolute()) {
- absoluteUri = parentUri;
- } else {
- absoluteUri = schema.getUri().resolve(parentUri);
- if (!absoluteUri.isAbsolute() || (absoluteUri.isAbsolute() && !schemaStore.getByUri(absoluteUri).isPresent() )) {
- absoluteUri = schema.getParentUri().resolve(parentUri);
- }
- }
- if (absoluteUri != null && absoluteUri.isAbsolute()) {
- Optional<Schema> schemaLookup = schemaStore.getByUri(absoluteUri);
- if (schemaLookup.isPresent()) {
- objectSchema = schemaLookup.get();
- }
- }
- }
- // have to resolve schema here
-
- builder = appendArrayObject(builder, objectSchema, fieldId, seperator);
- break;
- case ARRAY:
- ObjectNode subArrayItems = (ObjectNode) itemsNode.get("items");
- builder = appendArrayItems(builder, schema, fieldId, subArrayItems, seperator);
- break;
- default:
- builder = appendArrayField(builder, schema, fieldId, itemType, seperator);
- }
- } catch (Exception ex) {
- LOGGER.warn("No item type resolvable for {}", fieldId);
- }
- }
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendArrayField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- Objects.requireNonNull(fieldId);
- builder.append(cqlEscape(fieldId));
- builder.append(seperator);
- builder.append("list<" + cqlType(fieldType) + ">");
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendArrayObject(StringBuilder builder, Schema schema, String fieldId, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- String schemaSymbol = schemaSymbol(schema);
- if (StringUtils.isNotBlank(fieldId) && schemaSymbol != null ) {
- builder.append(cqlEscape(fieldId));
- builder.append(seperator);
- builder.append("list<" + schemaSymbol + ">");
- builder.append(LS);
- }
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendSchemaField(StringBuilder builder, Schema schema, String fieldId, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- String schemaSymbol = schemaSymbol(schema);
- if (StringUtils.isNotBlank(fieldId) && schemaSymbol != null ) {
- builder.append(cqlEscape(fieldId));
- builder.append(seperator);
- builder.append(schemaSymbol);
- }
- Objects.requireNonNull(builder);
- return builder;
- }
-
- /*
- can this be moved to streams-schemas if schemastore available in scope?
- maybe an interface?
- lot of boilerplate / reuse between plugins
- however treatment is way different when resolving a type symbol vs resolving and listing fields .
- */
- private StringBuilder appendPropertiesNode(StringBuilder builder, Schema schema, ObjectNode propertiesNode, Character seperator) {
- Objects.requireNonNull(builder);
- Objects.requireNonNull(propertiesNode);
- Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields();
- List<String> fieldStrings = new ArrayList<>();
- for ( ; fields.hasNext(); ) {
- Map.Entry<String, JsonNode> field = fields.next();
- String fieldId = field.getKey();
- if ( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) {
- ObjectNode fieldNode = (ObjectNode) field.getValue();
- FieldType fieldType = FieldUtil.determineFieldType(fieldNode);
- if (fieldType != null ) {
- switch (fieldType) {
- case ARRAY:
- ObjectNode itemsNode = (ObjectNode) fieldNode.get("items");
- if ( currentDepth <= config.getMaxDepth()) {
- StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, itemsNode, seperator);
- if (StringUtils.isNotBlank(arrayItemsBuilder.toString())) {
- fieldStrings.add(arrayItemsBuilder.toString());
- }
- }
- break;
- case OBJECT:
- Schema objectSchema = null;
- URI parentUri = null;
- if ( fieldNode.has("$ref") || fieldNode.has("extends") ) {
- JsonNode refNode = fieldNode.get("$ref");
- JsonNode extendsNode = fieldNode.get("extends");
- if (refNode != null && refNode.isValueNode()) {
- parentUri = URI.create(refNode.asText());
- } else if (extendsNode != null && extendsNode.isObject()) {
- parentUri = URI.create(extendsNode.get("$ref").asText());
- }
- URI absoluteUri;
- if (parentUri.isAbsolute()) {
- absoluteUri = parentUri;
- } else {
- absoluteUri = schema.getUri().resolve(parentUri);
- if (!absoluteUri.isAbsolute() || (absoluteUri.isAbsolute() && !schemaStore.getByUri(absoluteUri).isPresent() )) {
- absoluteUri = schema.getParentUri().resolve(parentUri);
- }
- }
- if (absoluteUri != null && absoluteUri.isAbsolute()) {
- Optional<Schema> schemaLookup = schemaStore.getByUri(absoluteUri);
- if (schemaLookup.isPresent()) {
- objectSchema = schemaLookup.get();
- }
- }
- }
- //ObjectNode childProperties = schemaStore.resolveProperties(schema, fieldNode, fieldId);
- if ( currentDepth < config.getMaxDepth()) {
- StringBuilder structFieldBuilder = appendSchemaField(new StringBuilder(), objectSchema, fieldId, seperator);
- if (StringUtils.isNotBlank(structFieldBuilder.toString())) {
- fieldStrings.add(structFieldBuilder.toString());
- }
- }
- break;
- default:
- StringBuilder valueFieldBuilder = appendValueField(new StringBuilder(), schema, fieldId, fieldType, seperator);
- if (StringUtils.isNotBlank(valueFieldBuilder.toString())) {
- fieldStrings.add(valueFieldBuilder.toString());
- }
- }
- }
- }
- }
- builder.append(String.join("," + LS, fieldStrings)).append(LS);
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private static String cqlEscape( String fieldId ) {
- return "`" + fieldId + "`";
- }
-
- private static String cqlType( FieldType fieldType ) {
- switch ( fieldType ) {
- case STRING:
- return "text";
- case INTEGER:
- return "int";
- case NUMBER:
- return "double";
- case OBJECT:
- return "tuple";
- case ARRAY:
- return "list";
- default:
- return fieldType.name().toUpperCase();
- }
- }
-
- private String schemaSymbol( Schema schema ) {
- if (schema == null) {
- return null;
- }
- // this needs to return whatever
- if (schema.getUri().getScheme().equals("file")) {
- String inputFile = schema.getUri().getPath();
- String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
- for (String sourcePath : config.getSourcePaths()) {
- resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
- }
- return dropExtension(resourcePath).replace("/", "_");
- } else {
- return "IDK";
- }
- }
-}
diff --git a/streams-plugins/streams-plugin-cassandra/src/main/java/org/apache/streams/plugins/cassandra/StreamsCassandraResourceGeneratorMojo.java b/streams-plugins/streams-plugin-cassandra/src/main/java/org/apache/streams/plugins/cassandra/StreamsCassandraResourceGeneratorMojo.java
deleted file mode 100644
index 3625f28..0000000
--- a/streams-plugins/streams-plugin-cassandra/src/main/java/org/apache/streams/plugins/cassandra/StreamsCassandraResourceGeneratorMojo.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.cassandra;
-
-import org.apache.maven.plugin.AbstractMojo;
-import org.apache.maven.plugin.MojoExecutionException;
-import org.apache.maven.plugin.MojoFailureException;
-import org.apache.maven.plugins.annotations.Component;
-import org.apache.maven.plugins.annotations.Execute;
-import org.apache.maven.plugins.annotations.LifecyclePhase;
-import org.apache.maven.plugins.annotations.Mojo;
-import org.apache.maven.plugins.annotations.Parameter;
-import org.apache.maven.project.MavenProject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.List;
-
-@Mojo (
- name = "generate-resources",
- defaultPhase = LifecyclePhase.GENERATE_RESOURCES
- )
-@Execute (
- goal = "generate-resources",
- phase = LifecyclePhase.GENERATE_RESOURCES
- )
-/**
- * Run within a module containing a src/main/jsonschema directory.
- *
- * <p/>
- * mvn org.apache.streams.plugins:streams-plugin-cassandra:0.4-incubating:cassandra
- *
- */
-public class StreamsCassandraResourceGeneratorMojo extends AbstractMojo {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsCassandraResourceGeneratorMojo.class);
-
- private volatile MojoFailureException mojoFailureException;
-
- @Component
- private MavenProject project;
-
- @Parameter( defaultValue = "${project.basedir}", readonly = true )
- private File basedir;
-
- @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only
- public String sourceDirectory;
-
- @Parameter( readonly = true ) // Maven 3 only
- public List<String> sourcePaths;
-
- @Parameter(defaultValue = "target/generated-resources/cassandra", readonly = true)
- public String targetDirectory;
-
- /**
- * execute StreamsCassandraResourceGenerator mojo.
- * @throws MojoExecutionException MojoExecutionException
- * @throws MojoFailureException MojoFailureException
- */
- public void execute() throws MojoExecutionException, MojoFailureException {
-
- //addProjectDependenciesToClasspath();
-
- StreamsCassandraGenerationConfig config = new StreamsCassandraGenerationConfig();
-
- if ( sourcePaths != null && sourcePaths.size() > 0) {
- config.setSourcePaths(sourcePaths);
- } else {
- config.setSourceDirectory(sourceDirectory);
- }
- config.setTargetDirectory(targetDirectory);
-
- StreamsCassandraResourceGenerator streamsCassandraResourceGenerator = new StreamsCassandraResourceGenerator(config);
-
- streamsCassandraResourceGenerator.run();
- }
-
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-cassandra/src/site/markdown/index.md b/streams-plugins/streams-plugin-cassandra/src/site/markdown/index.md
deleted file mode 100644
index 5300747..0000000
--- a/streams-plugins/streams-plugin-cassandra/src/site/markdown/index.md
+++ /dev/null
@@ -1,36 +0,0 @@
-org.apache.streams.plugins:streams-plugin-cassandra
-===================================================
-
-streams-plugin-cassandra generates resources from json schemas to assist with indexing of json data using Apache Cassandra.
-
-### Usage
-
-##### Maven
-
-Run within a module containing a src/main/jsonschema directory
-
- mvn org.apache.streams.plugins:streams-plugin-cassandra:0.4-incubating:cassandra
-
-[streams-plugin-cassandra/pom.xml](streams-plugin-cassandra/pom.xml "streams-plugin-cassandra/pom.xml")
-
-##### SDK
-
-Embed within your own java code
-
- StreamsCassandraGenerationConfig config = new StreamsCassandraGenerationConfig();
- config.setSourceDirectory("src/main/jsonschema");
- config.setTargetDirectory("target/generated-resources");
- StreamsCassandraResourceGenerator generator = new StreamsCassandraResourceGenerator(config);
- generator.run();
-
-##### CLI
-
-Run from CLI without Maven
-
- java -jar streams-plugin-cassandra-jar-with-dependencies.jar StreamsCassandraResourceGenerator src/main/jsonschema target/generated-resources
-
-#### Documentation
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorCLITest.java
deleted file mode 100644
index 4c79cd0..0000000
--- a/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorCLITest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.cassandra.test;
-
-import org.apache.streams.plugins.cassandra.StreamsCassandraResourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Collection;
-
-/**
- * Test that StreamsCassandraResourceGeneratorCLI generates resources.
- */
-public class StreamsCassandraResourceGeneratorCLITest {
-
- @Test
- public void testStreamsCassandraResourceGeneratorCLI() throws Exception {
-
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
- String targetDirectory = "target/generated-resources/test-cli";
-
- StreamsCassandraResourceGenerator.main(new String[]{sourceDirectory, targetDirectory});
-
- File testOutput = new File( targetDirectory );
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, StreamsCassandraResourceGeneratorTest.cqlFilter, true);
- Assert.assertEquals(outputCollection.size(), 1);
-
- Path path = Paths.get(testOutput.getAbsolutePath()).resolve("types.cql");
-
- Assert.assertTrue(path.toFile().exists());
-
- String typesCqlBytes = new String(java.nio.file.Files.readAllBytes(path));
-
- Assert.assertEquals(StringUtils.countMatches(typesCqlBytes, "CREATE TYPE"), 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorMojoIT.java b/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorMojoIT.java
deleted file mode 100644
index d35a0d3..0000000
--- a/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorMojoIT.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.cassandra.test;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.maven.it.Verifier;
-import org.apache.maven.it.util.ResourceExtractor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests that streams-plugin-cassandra running via maven generates cql resources.
- */
-public class StreamsCassandraResourceGeneratorMojoIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsCassandraResourceGeneratorMojoIT.class);
-
- @Test
- public void testStreamsCassandraResourceGeneratorMojo() throws Exception {
-
- File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-cassandra" );
-
- Verifier verifier;
-
- verifier = new Verifier( testDir.getAbsolutePath() );
-
- List<String> cliOptions = new ArrayList<>();
- cliOptions.add( "-N" );
- verifier.executeGoals(Stream.of(
- "clean",
- "dependency:unpack-dependencies",
- "generate-resources").collect(Collectors.toList()));
-
- verifier.verifyErrorFreeLog();
-
- verifier.resetStreams();
-
- Path testOutputPath = Paths.get(testDir.getAbsolutePath()).resolve("target/generated-resources/test-mojo");
-
- File testOutput = testOutputPath.toFile();
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, StreamsCassandraResourceGeneratorTest.cqlFilter, true);
- Assert.assertEquals(outputCollection.size(), 1);
-
- Path path = testOutputPath.resolve("types.cql");
-
- Assert.assertTrue(path.toFile().exists());
-
- String typesCqlBytes = new String(java.nio.file.Files.readAllBytes(path));
-
- Assert.assertEquals(StringUtils.countMatches(typesCqlBytes, "CREATE TYPE"), 133);
-
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java b/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java
deleted file mode 100644
index 19b0bb2..0000000
--- a/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.cassandra.test;
-
-import org.apache.streams.plugins.cassandra.StreamsCassandraGenerationConfig;
-import org.apache.streams.plugins.cassandra.StreamsCassandraResourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Collection;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Test that cassandra resources are generated.
- */
-public class StreamsCassandraResourceGeneratorTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsCassandraResourceGeneratorTest.class);
-
- public static final String[] cqlFilter = new String[]{"cql"};
-
- /**
- * Test that cassandra resources are generated.
- *
- * @throws Exception Exception
- */
- @Test
- public void testStreamsCassandraResourceGenerator() throws Exception {
-
- StreamsCassandraGenerationConfig config = new StreamsCassandraGenerationConfig();
-
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
-
- config.setSourceDirectory(sourceDirectory);
-
- config.setTargetDirectory("target/generated-resources/cassandra");
-
- config.setExclusions(Stream.of("attachments").collect(Collectors.toSet()));
-
- config.setMaxDepth(2);
-
- StreamsCassandraResourceGenerator streamsCassandraResourceGenerator = new StreamsCassandraResourceGenerator(config);
- streamsCassandraResourceGenerator.run();
-
- File testOutput = config.getTargetDirectory();
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, cqlFilter, true);
- Assert.assertEquals(outputCollection.size(), 1);
-
- Path path = Paths.get(testOutput.getAbsolutePath()).resolve("types.cql");
-
- Assert.assertTrue(path.toFile().exists());
-
- String typesCqlBytes = new String(java.nio.file.Files.readAllBytes(path));
-
- Assert.assertEquals(StringUtils.countMatches(typesCqlBytes, "CREATE TYPE"), 133);
-
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml b/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml
deleted file mode 100644
index 89056fa..0000000
--- a/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml
+++ /dev/null
@@ -1,143 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-cassandra-test</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Test StreamsCassandraResourceGeneratorMojo</name>
-
- <repositories>
- <repository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- </repositories>
-
- <pluginRepositories>
- <pluginRepository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </pluginRepository>
- <pluginRepository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </pluginRepository>
- </pluginRepositories>
-
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-cassandra</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <configuration>
- <sourcePaths>
- <sourcePath>target/test-classes/activitystreams-schemas/activity.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/collection.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/media_link.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/object.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/objectTypes</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/verbs</sourcePath>
- </sourcePaths>
- <targetDirectory>target/generated-resources/test-mojo</targetDirectory>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate-resources</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- </configuration>
- <executions>
- <execution>
- <id>unpack-schemas</id>
- <phase>process-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-elasticsearch/pom.xml b/streams-plugins/streams-plugin-elasticsearch/pom.xml
deleted file mode 100644
index 3351497..0000000
--- a/streams-plugins/streams-plugin-elasticsearch/pom.xml
+++ /dev/null
@@ -1,197 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
- <artifactId>streams-plugin-elasticsearch</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <packaging>maven-plugin</packaging>
-
- <parent>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugins</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <properties>
- <elasticsearch.version>2.3.0</elasticsearch.version>
- <lucene.version>5.4.0</lucene.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>commons-logging</artifactId>
- <groupId>commons-logging</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-aether-provider</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-artifact</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-compat</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-plugin-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.shared</groupId>
- <artifactId>maven-verifier</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.codehaus.plexus</groupId>
- <artifactId>plexus-utils</artifactId>
- <version>3.0.15</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-plugin-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-generators</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-testing</groupId>
- <artifactId>maven-plugin-testing-harness</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>org.joda</groupId>
- <artifactId>joda-convert</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <artifactId>maven-plugin-plugin</artifactId>
- <version>3.4</version>
- <configuration>
- <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
- </configuration>
- <executions>
- <execution>
- <id>mojo-descriptor</id>
- <phase>process-classes</phase>
- <goals>
- <goal>descriptor</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>resource-dependencies</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includes>**/*.json</includes>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
-</project>
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java
deleted file mode 100644
index 2a51a0c..0000000
--- a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.elasticsearch;
-
-import org.apache.streams.util.schema.GenerationConfig;
-
-import org.jsonschema2pojo.DefaultGenerationConfig;
-import org.jsonschema2pojo.util.URLUtil;
-
-import java.io.File;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Configures StreamsElasticsearchResourceGenerator.
- */
-public class StreamsElasticsearchGenerationConfig extends DefaultGenerationConfig implements GenerationConfig {
-
- public String getSourceDirectory() {
- return sourceDirectory;
- }
-
- public List<String> getSourcePaths() {
- return sourcePaths;
- }
-
- private String sourceDirectory;
- private List<String> sourcePaths = new ArrayList<String>();
- private String targetDirectory;
- private int maxDepth = 1;
-
- public Set<String> getExclusions() {
- return exclusions;
- }
-
- public void setExclusions(Set<String> exclusions) {
- this.exclusions = exclusions;
- }
-
- private Set<String> exclusions = new HashSet<String>();
-
- public int getMaxDepth() {
- return maxDepth;
- }
-
- public void setSourceDirectory(String sourceDirectory) {
- this.sourceDirectory = sourceDirectory;
- }
-
- public void setSourcePaths(List<String> sourcePaths) {
- this.sourcePaths = sourcePaths;
- }
-
- public void setTargetDirectory(String targetDirectory) {
- this.targetDirectory = targetDirectory;
- }
-
- public File getTargetDirectory() {
- return new File(targetDirectory);
- }
-
- /**
- * get all sources.
- * @return Iterator of URL
- */
- public Iterator<URL> getSource() {
- if (null != sourceDirectory) {
- return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator();
- }
- List<URL> sourceUrls = new ArrayList<URL>();
- if ( sourcePaths != null && sourcePaths.size() > 0) {
- for (String source : sourcePaths) {
- sourceUrls.add(URLUtil.parseURL(source));
- }
- }
- return sourceUrls.iterator();
- }
-
- public void setMaxDepth(int maxDepth) {
- this.maxDepth = maxDepth;
- }
-}
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java
deleted file mode 100644
index 9ce459e..0000000
--- a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java
+++ /dev/null
@@ -1,419 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.elasticsearch;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.schema.FieldType;
-import org.apache.streams.util.schema.FieldUtil;
-import org.apache.streams.util.schema.Schema;
-import org.apache.streams.util.schema.SchemaStore;
-import org.apache.streams.util.schema.SchemaStoreImpl;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang3.StringUtils;
-import org.jsonschema2pojo.util.URLUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.net.URI;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-
-import static org.apache.streams.util.schema.FileUtil.dropExtension;
-import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
-import static org.apache.streams.util.schema.FileUtil.resolveRecursive;
-import static org.apache.streams.util.schema.FileUtil.writeFile;
-
-public class StreamsElasticsearchResourceGenerator implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGenerator.class);
-
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- private static final String LS = System.getProperty("line.separator");
-
- private StreamsElasticsearchGenerationConfig config;
-
- private SchemaStore schemaStore = new SchemaStoreImpl();
-
- private int currentDepth = 0;
-
- /**
- * Run from CLI without Maven
- *
- * <p/>
- * java -jar streams-plugin-elasticsearch-jar-with-dependencies.jar StreamsElasticsearchResourceGenerator src/main/jsonschema target/generated-resources
- *
- * @param args [sourceDirectory, targetDirectory]
- */
- public static void main(String[] args) {
- StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
-
- String sourceDirectory = "src/main/jsonschema";
- String targetDirectory = "target/generated-resources/streams-plugin-elasticsearch";
-
- if ( args.length > 0 ) {
- sourceDirectory = args[0];
- }
- if ( args.length > 1 ) {
- targetDirectory = args[1];
- }
-
- config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory(targetDirectory);
-
- StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
- streamsElasticsearchResourceGenerator.run();
-
- }
-
- public StreamsElasticsearchResourceGenerator(StreamsElasticsearchGenerationConfig config) {
- this.config = config;
- }
-
- @Override
- public void run() {
-
- Objects.requireNonNull(config);
-
- generate(config);
-
- }
-
- /**
- * run generate using supplied StreamsElasticsearchGenerationConfig.
- * @param config StreamsElasticsearchGenerationConfig
- */
- public void generate(StreamsElasticsearchGenerationConfig config) {
-
- List<File> sourceFiles = new LinkedList<>();
-
- for (Iterator<URL> sources = config.getSource(); sources.hasNext();) {
- URL source = sources.next();
- sourceFiles.add(URLUtil.getFileFromURL(source));
- }
-
- LOGGER.info("Seeded with {} source paths:", sourceFiles.size());
-
- resolveRecursive(config, sourceFiles);
-
- LOGGER.info("Resolved {} schema files:", sourceFiles.size());
-
- for (File item : sourceFiles) {
- schemaStore.create(item.toURI());
- }
-
- LOGGER.info("Identified {} objects:", schemaStore.getSize());
-
- for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) {
- Schema schema = schemaIterator.next();
- currentDepth = 0;
- if ( schema.getUri().getScheme().equals("file")) {
- String inputFile = schema.getUri().getPath();
- String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
- for (String sourcePath : config.getSourcePaths()) {
- resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
- }
- String outputFile = config.getTargetDirectory() + "/" + resourcePath;
-
- LOGGER.info("Processing {}:", resourcePath);
-
- String resourceId = schemaSymbol(schema);
-
- String resourceContent = generateResource(schema, resourceId);
-
- if (StringUtils.isNotBlank(resourceContent)) {
- writeFile(outputFile, resourceContent);
- }
-
- LOGGER.info("Wrote {}:", outputFile);
- }
- }
-
- }
-
- /**
- * generateResource String from schema and resourceId.
- * @param schema Schema
- * @param resourceId String
- * @return mapping
- */
- public String generateResource(Schema schema, String resourceId) {
- StringBuilder resourceBuilder = new StringBuilder();
-
- ObjectNode rootNode = (ObjectNode) schema.getContent();
-
- // remove java*
- // remove description
- // resolve all $ref
- // replace format: date with type: date
- // replace format: date-time with type: date
- // replace array of primitive with just primitive
-
- try {
- String objectString = MAPPER.writeValueAsString(rootNode);
- resourceBuilder.append(objectString);
- } catch (JsonProcessingException ex) {
- LOGGER.error("{}: {}", ex.getClass().getName(), ex);
- }
- return resourceBuilder.toString();
- }
-
- protected StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId, Character seperator) {
- ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId);
- if ( propertiesNode.get("id") != null ) {
- builder.append("id text PRIMARY KEY,");
- builder.append(LS);
- propertiesNode.remove("id");
- }
- if ( propertiesNode.isObject() && propertiesNode.size() > 0) {
- builder = appendPropertiesNode(builder, schema, propertiesNode, seperator);
- }
- return builder;
- }
-
- private StringBuilder appendValueField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- builder.append(cqlEscape(fieldId));
- builder.append(seperator);
- builder.append(cqlType(fieldType));
- return builder;
- }
-
- protected StringBuilder appendArrayItems(StringBuilder builder, Schema schema, String fieldId, ObjectNode itemsNode, Character seperator) {
- // not safe to append nothing
- Objects.requireNonNull(builder);
- if ( itemsNode == null ) {
- return builder;
- }
- if ( itemsNode.has("type")) {
- try {
- FieldType itemType = FieldUtil.determineFieldType(itemsNode);
- switch ( itemType ) {
- case OBJECT:
- Schema objectSchema = null;
- URI parentUri = null;
- if ( itemsNode.has("$ref") || itemsNode.has("extends") ) {
- JsonNode refNode = itemsNode.get("$ref");
- JsonNode extendsNode = itemsNode.get("extends");
- if (refNode != null && refNode.isValueNode()) {
- parentUri = URI.create(refNode.asText());
- } else if (extendsNode != null && extendsNode.isObject()) {
- parentUri = URI.create(extendsNode.get("$ref").asText());
- }
- URI absoluteUri;
- if (parentUri.isAbsolute()) {
- absoluteUri = parentUri;
- } else {
- absoluteUri = schema.getUri().resolve(parentUri);
- if (!absoluteUri.isAbsolute() || (absoluteUri.isAbsolute() && !schemaStore.getByUri(absoluteUri).isPresent() )) {
- absoluteUri = schema.getParentUri().resolve(parentUri);
- }
- }
- if (absoluteUri.isAbsolute()) {
- Optional<Schema> schemaLookup = schemaStore.getByUri(absoluteUri);
- if (schemaLookup.isPresent()) {
- objectSchema = schemaLookup.get();
- }
- }
- }
- // have to resolve schema here
-
- builder = appendArrayObject(builder, objectSchema, fieldId, seperator);
- break;
- case ARRAY:
- ObjectNode subArrayItems = (ObjectNode) itemsNode.get("items");
- builder = appendArrayItems(builder, schema, fieldId, subArrayItems, seperator);
- break;
- default:
- builder = appendArrayField(builder, schema, fieldId, itemType, seperator);
- }
- } catch (Exception ex) {
- LOGGER.warn("No item type resolvable for {}", fieldId);
- }
- }
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendArrayField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- Objects.requireNonNull(fieldId);
- builder.append(cqlEscape(fieldId));
- builder.append(seperator);
- builder.append("list<").append(cqlType(fieldType)).append(">");
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendArrayObject(StringBuilder builder, Schema schema, String fieldId, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- String schemaSymbol = schemaSymbol(schema);
- if (StringUtils.isNotBlank(fieldId) && schemaSymbol != null ) {
- builder.append(cqlEscape(fieldId));
- builder.append(seperator);
- builder.append("list<").append(schemaSymbol).append(">");
- builder.append(LS);
- }
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendSchemaField(StringBuilder builder, Schema schema, String fieldId, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- String schemaSymbol = schemaSymbol(schema);
- if (StringUtils.isNotBlank(fieldId) && schemaSymbol != null ) {
- builder.append(cqlEscape(fieldId));
- builder.append(seperator);
- builder.append(schemaSymbol);
- }
- Objects.requireNonNull(builder);
- return builder;
- }
-
- /*
- can this be moved to streams-schemas if schemastore available in scope?
- maybe an interface?
- lot of boilerplate / reuse between plugins
- however treatment is way different when resolving a type symbol vs resolving and listing fields .
- */
- private StringBuilder appendPropertiesNode(StringBuilder builder, Schema schema, ObjectNode propertiesNode, Character seperator) {
- Objects.requireNonNull(builder);
- Objects.requireNonNull(propertiesNode);
- Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields();
- List<String> fieldStrings = new ArrayList<>();
- for ( ; fields.hasNext(); ) {
- Map.Entry<String, JsonNode> field = fields.next();
- String fieldId = field.getKey();
- if ( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) {
- ObjectNode fieldNode = (ObjectNode) field.getValue();
- FieldType fieldType = FieldUtil.determineFieldType(fieldNode);
- if (fieldType != null ) {
- switch (fieldType) {
- case ARRAY:
- ObjectNode itemsNode = (ObjectNode) fieldNode.get("items");
- if ( currentDepth <= config.getMaxDepth()) {
- StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, itemsNode, seperator);
- if (StringUtils.isNotBlank(arrayItemsBuilder.toString())) {
- fieldStrings.add(arrayItemsBuilder.toString());
- }
- }
- break;
- case OBJECT:
- Schema objectSchema = null;
- URI parentUri = null;
- if ( fieldNode.has("$ref") || fieldNode.has("extends") ) {
- JsonNode refNode = fieldNode.get("$ref");
- JsonNode extendsNode = fieldNode.get("extends");
- if (refNode != null && refNode.isValueNode()) {
- parentUri = URI.create(refNode.asText());
- } else if (extendsNode != null && extendsNode.isObject()) {
- parentUri = URI.create(extendsNode.get("$ref").asText());
- }
- URI absoluteUri;
- if (parentUri.isAbsolute()) {
- absoluteUri = parentUri;
- } else {
- absoluteUri = schema.getUri().resolve(parentUri);
- if (!absoluteUri.isAbsolute() || (absoluteUri.isAbsolute() && !schemaStore.getByUri(absoluteUri).isPresent() )) {
- absoluteUri = schema.getParentUri().resolve(parentUri);
- }
- }
- if (absoluteUri.isAbsolute()) {
- Optional<Schema> schemaLookup = schemaStore.getByUri(absoluteUri);
- if (schemaLookup.isPresent()) {
- objectSchema = schemaLookup.get();
- }
- }
- }
- //ObjectNode childProperties = schemaStore.resolveProperties(schema, fieldNode, fieldId);
- if ( currentDepth < config.getMaxDepth()) {
- StringBuilder structFieldBuilder = appendSchemaField(new StringBuilder(), objectSchema, fieldId, seperator);
- if (StringUtils.isNotBlank(structFieldBuilder.toString())) {
- fieldStrings.add(structFieldBuilder.toString());
- }
- }
- break;
- default:
- StringBuilder valueFieldBuilder = appendValueField(new StringBuilder(), schema, fieldId, fieldType, seperator);
- if (StringUtils.isNotBlank(valueFieldBuilder.toString())) {
- fieldStrings.add(valueFieldBuilder.toString());
- }
- }
- }
- }
- }
- builder.append(String.join("," + LS, fieldStrings)).append(LS);
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private static String cqlEscape( String fieldId ) {
- return "`" + fieldId + "`";
- }
-
- private static String cqlType( FieldType fieldType ) {
- switch ( fieldType ) {
- case STRING:
- return "text";
- case INTEGER:
- return "int";
- case NUMBER:
- return "double";
- case OBJECT:
- return "tuple";
- case ARRAY:
- return "list";
- default:
- return fieldType.name().toUpperCase();
- }
- }
-
- private String schemaSymbol( Schema schema ) {
- if (schema == null) {
- return null;
- }
- // this needs to return whatever
- if (schema.getUri().getScheme().equals("file")) {
- String inputFile = schema.getUri().getPath();
- String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
- for (String sourcePath : config.getSourcePaths()) {
- resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
- }
- return dropExtension(resourcePath).replace("/", "_");
- } else {
- return "IDK";
- }
- }
-}
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
deleted file mode 100644
index ea512b3..0000000
--- a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.elasticsearch;
-
-import org.apache.maven.plugin.AbstractMojo;
-import org.apache.maven.plugin.MojoExecutionException;
-import org.apache.maven.plugin.MojoFailureException;
-import org.apache.maven.plugins.annotations.Component;
-import org.apache.maven.plugins.annotations.Execute;
-import org.apache.maven.plugins.annotations.LifecyclePhase;
-import org.apache.maven.plugins.annotations.Mojo;
-import org.apache.maven.plugins.annotations.Parameter;
-import org.apache.maven.project.MavenProject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.List;
-
-@Mojo ( name = "generate-resources",
- defaultPhase = LifecyclePhase.GENERATE_RESOURCES
- )
-@Execute ( goal = "generate-resources",
- phase = LifecyclePhase.GENERATE_RESOURCES
- )
-public class StreamsElasticsearchResourceGeneratorMojo extends AbstractMojo {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojo.class);
-
- private volatile MojoFailureException mojoFailureException;
-
- @Component
- private MavenProject project;
-
- // @Component
- // private Settings settings;
- //
- // @Parameter( defaultValue = "${localRepository}", readonly = true, required = true )
- // protected ArtifactRepository localRepository;
- //
- // @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only
- // private PluginDescriptor plugin;
- //
- @Parameter( defaultValue = "${project.basedir}", readonly = true )
- private File basedir;
-
- @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only
- public String sourceDirectory;
-
- @Parameter( readonly = true ) // Maven 3 only
- public List<String> sourcePaths;
-
- @Parameter(defaultValue = "./target/generated-resources/streams-plugin-elasticsearch", readonly = true)
- public String targetDirectory;
-
- public void execute() throws MojoExecutionException, MojoFailureException {
-
- //addProjectDependenciesToClasspath();
-
- StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
-
- if ( sourcePaths != null && sourcePaths.size() > 0) {
- config.setSourcePaths(sourcePaths);
- } else {
- config.setSourceDirectory(sourceDirectory);
- }
- config.setTargetDirectory(targetDirectory);
-
- StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
-
- streamsElasticsearchResourceGenerator.run();
-
- }
-
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md b/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md
deleted file mode 100644
index 4b5af8e..0000000
--- a/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md
+++ /dev/null
@@ -1,38 +0,0 @@
-org.apache.streams.plugins:streams-plugin-elasticsearch
-=======================================================
-
-streams-plugin-elasticsearch generates resources from json schemas to assist with indexing of json data using Elasticsearch.
-
-### Usage
-
-Output will be placed in target/generated-resources/elasticsearch by default
-
-##### Maven
-
-Run within a module containing a src/main/jsonschema directory
-
- mvn org.apache.streams.plugins:streams-plugin-elasticsearch:0.4-incubating:elasticsearch
-
-[streams-plugin-elasticsearch/pom.xml](streams-plugin-elasticsearch/pom.xml "streams-plugin-elasticsearch/pom.xml")
-
-##### SDK
-
-Embed within your own java code
-
- StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
- config.setSourceDirectory("src/main/jsonschema");
- config.setTargetDirectory("target/generated-resources");
- StreamsElasticsearchResourceGenerator generator = new StreamsElasticsearchResourceGenerator(config);
- generator.run();
-
-##### CLI
-
-Run from CLI without Maven
-
- java -jar streams-plugin-elasticsearch-jar-with-dependencies.jar StreamsElasticsearchResourceGenerator src/main/jsonschema target/generated-resources
-
-#### Documentation
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java
deleted file mode 100644
index a2cddbb..0000000
--- a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.elasticsearch.test;
-
-import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-import java.util.Collection;
-
-/**
- * Test that StreamsElasticsearchResourceGeneratorCLI generates resources.
- */
-public class StreamsElasticsearchResourceGeneratorCLITest {
-
- @Test
- public void testStreamsElasticsearchResourceGeneratorCLI() throws Exception {
-
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
- String targetDirectory = "target/generated-resources/elasticsearch-cli";
-
- StreamsElasticsearchResourceGenerator.main(new String[]{sourceDirectory, targetDirectory});
-
- File testOutput = new File(targetDirectory);
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, StreamsElasticsearchResourceGeneratorTest.jsonFilter, true);
- Assert.assertEquals(outputCollection.size(), 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java
deleted file mode 100644
index 3b24d09..0000000
--- a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.elasticsearch.test;
-
-import org.apache.maven.it.Verifier;
-import org.apache.maven.it.util.ResourceExtractor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests that streams-plugin-elasticsearch running via maven generates elasticsearch mapping resources.
- */
-public class StreamsElasticsearchResourceGeneratorMojoIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojoIT.class);
-
- @Test
- public void testStreamsElasticsearchResourceGeneratorMojo() throws Exception {
-
- File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-elasticsearch" );
-
- Verifier verifier;
-
- verifier = new Verifier( testDir.getAbsolutePath() );
-
- List<String> cliOptions = new ArrayList<>();
- cliOptions.add( "-N" );
- verifier.executeGoals(Stream.of(
- "clean",
- "dependency:unpack-dependencies",
- "generate-resources").collect(Collectors.toList()));
-
- verifier.verifyErrorFreeLog();
-
- verifier.resetStreams();
-
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
deleted file mode 100644
index 3950aa5..0000000
--- a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.elasticsearch.test;
-
-import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchGenerationConfig;
-import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.Collection;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Test that Elasticsearch resources are generated.
- */
-public class StreamsElasticsearchResourceGeneratorTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorTest.class);
-
- public static final String[] jsonFilter = new String[]{"json"};
-
- /**
- * Test that Elasticsearch resources are generated.
- *
- * @throws Exception Exception
- */
- @Test
- public void StreamsElasticsearchResourceGenerator() throws Exception {
-
- StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
-
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
-
- config.setSourceDirectory(sourceDirectory);
-
- config.setTargetDirectory("target/generated-resources/elasticsearch");
-
- config.setExclusions(Stream.of("attachments").collect(Collectors.toSet()));
-
- config.setMaxDepth(2);
-
- StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
- streamsElasticsearchResourceGenerator.run();
-
- File testOutput = config.getTargetDirectory();
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, jsonFilter, true);
- Assert.assertEquals(outputCollection.size(), 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml b/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml
deleted file mode 100644
index 87b536d..0000000
--- a/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml
+++ /dev/null
@@ -1,143 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-elasticsearch-test</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Test StreamsElasticsearchResourceGeneratorMojo</name>
-
- <repositories>
- <repository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- </repositories>
-
- <pluginRepositories>
- <pluginRepository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </pluginRepository>
- <pluginRepository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </pluginRepository>
- </pluginRepositories>
-
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-elasticsearch</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <configuration>
- <sourcePaths>
- <sourcePath>target/test-classes/activitystreams-schemas/activity.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/collection.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/media_link.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/object.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/objectTypes</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/verbs</sourcePath>
- </sourcePaths>
- <targetDirectory>target/generated-resources/elasticsearch-mojo</targetDirectory>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate-resources</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- </configuration>
- <executions>
- <execution>
- <id>unpack-schemas</id>
- <phase>process-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hbase/pom.xml b/streams-plugins/streams-plugin-hbase/pom.xml
deleted file mode 100644
index 18b3b3d..0000000
--- a/streams-plugins/streams-plugin-hbase/pom.xml
+++ /dev/null
@@ -1,196 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
- <artifactId>streams-plugin-hbase</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <packaging>maven-plugin</packaging>
-
- <parent>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugins</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <properties>
- <hbase.version>3.0</hbase.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>commons-logging</artifactId>
- <groupId>commons-logging</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-aether-provider</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-artifact</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-compat</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-plugin-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.shared</groupId>
- <artifactId>maven-verifier</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.codehaus.plexus</groupId>
- <artifactId>plexus-utils</artifactId>
- <version>3.0.15</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-plugin-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-generators</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-testing</groupId>
- <artifactId>maven-plugin-testing-harness</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>org.joda</groupId>
- <artifactId>joda-convert</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <artifactId>maven-plugin-plugin</artifactId>
- <version>3.4</version>
- <configuration>
- <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
- </configuration>
- <executions>
- <execution>
- <id>mojo-descriptor</id>
- <phase>process-classes</phase>
- <goals>
- <goal>descriptor</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>resource-dependencies</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includes>**/*.json</includes>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
-</project>
diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
deleted file mode 100644
index e4c8943..0000000
--- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.hbase;
-
-import org.apache.streams.util.schema.GenerationConfig;
-
-import org.jsonschema2pojo.DefaultGenerationConfig;
-import org.jsonschema2pojo.util.URLUtil;
-
-import java.io.File;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Configures StreamsHiveResourceGenerator.
- */
-public class StreamsHbaseGenerationConfig extends DefaultGenerationConfig implements GenerationConfig {
-
- public String getSourceDirectory() {
- return sourceDirectory;
- }
-
- public List<String> getSourcePaths() {
- return sourcePaths;
- }
-
- private String columnFamily;
- private String sourceDirectory;
- private List<String> sourcePaths = new ArrayList<>();
- private String targetDirectory;
- private int maxDepth = 1;
-
- public Set<String> getExclusions() {
- return exclusions;
- }
-
- public void setExclusions(Set<String> exclusions) {
- this.exclusions = exclusions;
- }
-
- private Set<String> exclusions = new HashSet<>();
-
- public int getMaxDepth() {
- return maxDepth;
- }
-
- public void setSourceDirectory(String sourceDirectory) {
- this.sourceDirectory = sourceDirectory;
- }
-
- public void setSourcePaths(List<String> sourcePaths) {
- this.sourcePaths = sourcePaths;
- }
-
- public void setTargetDirectory(String targetDirectory) {
- this.targetDirectory = targetDirectory;
- }
-
- public File getTargetDirectory() {
- return new File(targetDirectory);
- }
-
- /**
- * get all sources.
- * @return Iterator of URL
- */
- public Iterator<URL> getSource() {
- if (null != sourceDirectory) {
- return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator();
- }
- List<URL> sourceUrls = new ArrayList<>();
- if ( sourcePaths != null && sourcePaths.size() > 0) {
- for (String source : sourcePaths) {
- sourceUrls.add(URLUtil.parseURL(source));
- }
- }
- return sourceUrls.iterator();
- }
-
- public void setMaxDepth(int maxDepth) {
- this.maxDepth = maxDepth;
- }
-
- public String getColumnFamily() {
- return columnFamily;
- }
-
- public void setColumnFamily(String columnFamily) {
- this.columnFamily = columnFamily;
- }
-}
diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
deleted file mode 100644
index 0701a6f..0000000
--- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.hbase;
-
-import org.apache.streams.util.schema.FieldType;
-import org.apache.streams.util.schema.FieldUtil;
-import org.apache.streams.util.schema.Schema;
-import org.apache.streams.util.schema.SchemaStore;
-import org.apache.streams.util.schema.SchemaStoreImpl;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.jsonschema2pojo.util.URLUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.apache.streams.util.schema.FileUtil.dropExtension;
-import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
-import static org.apache.streams.util.schema.FileUtil.resolveRecursive;
-import static org.apache.streams.util.schema.FileUtil.swapExtension;
-import static org.apache.streams.util.schema.FileUtil.writeFile;
-
-public class StreamsHbaseResourceGenerator implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGenerator.class);
-
- private static final String LS = System.getProperty("line.separator");
-
- private StreamsHbaseGenerationConfig config;
-
- private SchemaStore schemaStore = new SchemaStoreImpl();
-
- private int currentDepth = 0;
-
- /**
- * Run from CLI without Maven
- *
- * <p/>
- * java -jar streams-plugin-hbase-jar-with-dependencies.jar StreamsHbaseResourceGenerator src/main/jsonschema target/generated-resources
- *
- * @param args [sourceDirectory, targetDirectory]
- * */
- public static void main(String[] args) {
- StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
-
- String sourceDirectory = "src/main/jsonschema";
- String targetDirectory = "target/generated-resources/hbase";
-
- if ( args.length > 0 ) {
- sourceDirectory = args[0];
- }
- if ( args.length > 1 ) {
- targetDirectory = args[1];
- }
-
- config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory(targetDirectory);
-
- StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
- streamsHbaseResourceGenerator.run();
-
- }
-
- public StreamsHbaseResourceGenerator(StreamsHbaseGenerationConfig config) {
- this.config = config;
- }
-
- @Override
- public void run() {
-
- Objects.requireNonNull(config);
-
- generate(config);
-
- }
-
- /**
- * run generate using supplied StreamsHbaseGenerationConfig.
- * @param config StreamsHbaseGenerationConfig
- */
- public void generate(StreamsHbaseGenerationConfig config) {
-
- LinkedList<File> sourceFiles = new LinkedList<>();
-
- for (Iterator<URL> sources = config.getSource(); sources.hasNext();) {
- URL source = sources.next();
- sourceFiles.add(URLUtil.getFileFromURL(source));
- }
-
- LOGGER.info("Seeded with {} source paths:", sourceFiles.size());
-
- resolveRecursive(config, sourceFiles);
-
- LOGGER.info("Resolved {} schema files:", sourceFiles.size());
-
- for (File item : sourceFiles) {
- schemaStore.create(item.toURI());
- }
-
- LOGGER.info("Identified {} objects:", schemaStore.getSize());
-
- for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) {
- Schema schema = schemaIterator.next();
- currentDepth = 0;
- if ( schema.getUri().getScheme().equals("file")) {
- String inputFile = schema.getUri().getPath();
- String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
- for (String sourcePath : config.getSourcePaths()) {
- resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
- }
- String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "txt");
-
- LOGGER.info("Processing {}:", resourcePath);
-
- String resourceId = dropExtension(resourcePath).replace("/", "_");
-
- String resourceContent = generateResource(schema, resourceId);
-
- writeFile(outputFile, resourceContent);
-
- LOGGER.info("Wrote {}:", outputFile);
- }
- }
-
- }
-
- /**
- * generateResource String from schema and resourceId.
- * @param schema Schema
- * @param resourceId String
- * @return mapping
- */
- public String generateResource(Schema schema, String resourceId) {
- StringBuilder resourceBuilder = new StringBuilder();
- resourceBuilder.append("CREATE ");
- resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId);
- return resourceBuilder.toString();
- }
-
- protected StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId) {
- Objects.requireNonNull(builder);
- ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId);
- if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) {
-
- List<String> fieldStrings = new ArrayList<>();
-
- // table
- fieldStrings.add(hbaseEscape(schemaSymbol(schema)));
-
- // column family
- fieldStrings.add(hbaseEscape(schemaSymbol(schema)));
-
- // parent column family
- if ( schema.getParent() != null ) {
- fieldStrings.add(hbaseEscape(schemaSymbol(schema.getParent())));
- }
-
- // sub-object column families
- if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) {
-
- Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields();
- for ( ; fields.hasNext(); ) {
- Map.Entry<String, JsonNode> field = fields.next();
- String fieldId = field.getKey();
- if ( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) {
- ObjectNode fieldNode = (ObjectNode) field.getValue();
- FieldType fieldType = FieldUtil.determineFieldType(fieldNode);
- if (fieldType != null ) {
- switch (fieldType) {
- case OBJECT:
- fieldStrings.add(hbaseEscape(fieldId));
- break;
- default:
- break;
- }
- }
- }
- }
- builder.append(String.join(", ", fieldStrings));
-
- }
- }
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private static String hbaseEscape( String fieldId ) {
- return "'" + fieldId + "'";
- }
-
- private String schemaSymbol( Schema schema ) {
- if (schema == null) {
- return null;
- }
- if (schema.getUri().getScheme().equals("file")) {
- String inputFile = schema.getUri().getPath();
- String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
- for (String sourcePath : config.getSourcePaths()) {
- resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
- }
- return dropExtension(resourcePath).replace("/", "_");
- } else {
- return "IDK";
- }
- }
-
-}
diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
deleted file mode 100644
index addc657..0000000
--- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.hbase;
-
-import org.apache.maven.plugin.AbstractMojo;
-import org.apache.maven.plugin.MojoExecutionException;
-import org.apache.maven.plugin.MojoFailureException;
-import org.apache.maven.plugins.annotations.Component;
-import org.apache.maven.plugins.annotations.Execute;
-import org.apache.maven.plugins.annotations.LifecyclePhase;
-import org.apache.maven.plugins.annotations.Mojo;
-import org.apache.maven.plugins.annotations.Parameter;
-import org.apache.maven.project.MavenProject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.List;
-
-@Mojo (
- name = "generate-resources",
- defaultPhase = LifecyclePhase.GENERATE_RESOURCES
- )
-@Execute (
- goal = "generate-resources",
- phase = LifecyclePhase.GENERATE_RESOURCES
- )
-public class StreamsHbaseResourceGeneratorMojo extends AbstractMojo {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojo.class);
-
- private volatile MojoFailureException mojoFailureException;
-
- @Component
- private MavenProject project;
-
- // @Component
- // private Settings settings;
- //
- // @Parameter( defaultValue = "${localRepository}", readonly = true, required = true )
- // protected ArtifactRepository localRepository;
- //
- // @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only
- // private PluginDescriptor plugin;
- //
- @Parameter( defaultValue = "${project.basedir}", readonly = true )
- private File basedir;
-
- @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only
- public String sourceDirectory;
-
- @Parameter( readonly = true ) // Maven 3 only
- public List<String> sourcePaths;
-
- @Parameter(defaultValue = "./target/generated-resources/hbase", readonly = true)
- public String targetDirectory;
-
- /**
- * execute StreamsHbaseResourceGenerator mojo.
- * @throws MojoExecutionException MojoExecutionException
- * @throws MojoFailureException MojoFailureException
- */
- public void execute() throws MojoExecutionException, MojoFailureException {
-
- //addProjectDependenciesToClasspath();
-
- StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
-
- if ( sourcePaths != null && sourcePaths.size() > 0) {
- config.setSourcePaths(sourcePaths);
- } else {
- config.setSourceDirectory(sourceDirectory);
- }
- config.setTargetDirectory(targetDirectory);
-
- StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
- streamsHbaseResourceGenerator.run();
-
- }
-
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md b/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md
deleted file mode 100644
index 5ac8ef3..0000000
--- a/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md
+++ /dev/null
@@ -1,40 +0,0 @@
-org.apache.streams.plugins:streams-plugin-hbase
-===============================================
-
-streams-plugin-hbase generates resources from json schemas to assist with indexing of json data using Apache HBase.
-
-### Usage
-
-Output will be placed in target/generated-resources/hbase by default
-
-[streams-plugin-hbase/pom.xml](streams-plugin-hbase/pom.xml "streams-plugin-hbase/pom.xml")
-
-##### Maven
-
-Run within a module containing a src/main/jsonschema directory
-
- mvn org.apache.streams.plugins:streams-plugin-hbase:0.4-incubating:hbase
-
-##### SDK
-
-Embed within your own java code
-
- StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
- config.setSourceDirectory("src/main/jsonschema");
- config.setTargetDirectory("target/generated-resources");
- StreamsHbaseResourceGenerator generator = new StreamsHbaseResourceGenerator(config);
- generator.run();
-
-##### CLI
-
-Run from CLI without Maven
-
- java -jar streams-plugin-hbase-jar-with-dependencies.jar StreamsHbaseResourceGenerator src/main/jsonschema target/generated-resources
-
-#### Example
-
-#### Documentation
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
deleted file mode 100644
index fde1ded..0000000
--- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.test;
-
-import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-import java.util.Collection;
-
-/**
- * Test that StreamsHbaseResourceGeneratorCLI generates resources.
- */
-public class StreamsHbaseResourceGeneratorCLITest {
-
- @Test
- public void testStreamsHbaseResourceGeneratorCLI() throws Exception {
-
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
- String targetDirectory = "target/generated-resources/hbase-cli";
-
- StreamsHbaseResourceGenerator.main(new String[]{sourceDirectory, targetDirectory});
-
- File testOutput = new File(targetDirectory);
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, StreamsHbaseResourceGeneratorTest.txtFilter, true);
- Assert.assertEquals(outputCollection.size(), 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java
deleted file mode 100644
index 82d9991..0000000
--- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.test;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.maven.it.Verifier;
-import org.apache.maven.it.util.ResourceExtractor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests that streams-plugin-hbase running via maven generates txt resources.
- */
-public class StreamsHbaseResourceGeneratorMojoIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojoIT.class);
-
- @Test
- public void testStreamsHbaseResourceGeneratorMojo() throws Exception {
-
- File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-hbase" );
-
- Verifier verifier;
-
- verifier = new Verifier( testDir.getAbsolutePath() );
-
- List<String> cliOptions = new ArrayList<>();
- cliOptions.add( "-N" );
- verifier.executeGoals(Stream.of(
- "clean",
- "dependency:unpack-dependencies",
- "generate-resources").collect(Collectors.toList()));
-
- verifier.verifyErrorFreeLog();
-
- verifier.resetStreams();
-
- File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/hbase-mojo");
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, StreamsHbaseResourceGeneratorTest.txtFilter, true);
- Assert.assertEquals(outputCollection.size(), 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
deleted file mode 100644
index 13fdfb3..0000000
--- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.test;
-
-import org.apache.streams.plugins.hbase.StreamsHbaseGenerationConfig;
-import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.Collection;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Test that Activity beans are compatible with the example activities in the spec.
- */
-public class StreamsHbaseResourceGeneratorTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorTest.class);
-
- public static final String[] txtFilter = new String[]{"txt"};
-
- /**
- * Tests that all example activities can be loaded into Activity beans.
- *
- * @throws Exception Exception
- */
- @Test
- public void testStreamsHbaseResourceGenerator() throws Exception {
-
- StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
-
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
-
- config.setSourceDirectory(sourceDirectory);
-
- config.setTargetDirectory("target/generated-resources/hbase");
-
- config.setExclusions(Stream.of("attachments").collect(Collectors.toSet()));
-
- config.setColumnFamily("cf");
- config.setMaxDepth(2);
-
- StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
- streamsHbaseResourceGenerator.run();
-
- File testOutput = config.getTargetDirectory();
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, txtFilter, true);
- Assert.assertEquals(outputCollection.size(), 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml b/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml
deleted file mode 100644
index 67799a7..0000000
--- a/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml
+++ /dev/null
@@ -1,145 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-hbase-test</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Test StreamsHbaseResourceGeneratorMojo</name>
-
- <repositories>
- <repository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- </repositories>
-
- <pluginRepositories>
- <pluginRepository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </pluginRepository>
- <pluginRepository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </pluginRepository>
- </pluginRepositories>
-
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-hbase</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <configuration>
- <sourcePaths>
- <sourcePath>target/test-classes/activitystreams-schemas/activity.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/collection.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/media_link.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/object.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/objectTypes</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/verbs</sourcePath>
- </sourcePaths>
- <targetDirectory>target/generated-resources/hbase-mojo</targetDirectory>
- <columnFamily>cf</columnFamily>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate-resources</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- </configuration>
- <executions>
- <execution>
- <id>unpack-schemas</id>
- <phase>process-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hive/pom.xml b/streams-plugins/streams-plugin-hive/pom.xml
deleted file mode 100644
index 60aa4b4..0000000
--- a/streams-plugins/streams-plugin-hive/pom.xml
+++ /dev/null
@@ -1,196 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
- <artifactId>streams-plugin-hive</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <packaging>maven-plugin</packaging>
-
- <parent>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugins</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <properties>
- <hive.version>1.14</hive.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>commons-logging</artifactId>
- <groupId>commons-logging</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-aether-provider</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-artifact</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-compat</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-plugin-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.codehaus.plexus</groupId>
- <artifactId>plexus-utils</artifactId>
- <version>3.0.15</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-plugin-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-generators</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-testing</groupId>
- <artifactId>maven-plugin-testing-harness</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.shared</groupId>
- <artifactId>maven-verifier</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>org.joda</groupId>
- <artifactId>joda-convert</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <artifactId>maven-plugin-plugin</artifactId>
- <version>3.4</version>
- <configuration>
- <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
- </configuration>
- <executions>
- <execution>
- <id>mojo-descriptor</id>
- <phase>process-classes</phase>
- <goals>
- <goal>descriptor</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>resource-dependencies</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includes>**/*.json</includes>
- <includeTypes>jar</includeTypes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
-</project>
diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java
deleted file mode 100644
index cc32255..0000000
--- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.hive;
-
-import org.apache.streams.util.schema.GenerationConfig;
-
-import org.jsonschema2pojo.DefaultGenerationConfig;
-import org.jsonschema2pojo.util.URLUtil;
-
-import java.io.File;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Configures StreamsHiveResourceGenerator.
- */
-public class StreamsHiveGenerationConfig extends DefaultGenerationConfig implements GenerationConfig {
-
- public String getSourceDirectory() {
- return sourceDirectory;
- }
-
- public List<String> getSourcePaths() {
- return sourcePaths;
- }
-
- private String sourceDirectory;
- private List<String> sourcePaths = new ArrayList<String>();
- private String targetDirectory;
- private int maxDepth = 1;
-
- public Set<String> getExclusions() {
- return exclusions;
- }
-
- public void setExclusions(Set<String> exclusions) {
- this.exclusions = exclusions;
- }
-
- private Set<String> exclusions = new HashSet<String>();
-
- public int getMaxDepth() {
- return maxDepth;
- }
-
- public void setSourceDirectory(String sourceDirectory) {
- this.sourceDirectory = sourceDirectory;
- }
-
- public void setSourcePaths(List<String> sourcePaths) {
- this.sourcePaths = sourcePaths;
- }
-
- public void setTargetDirectory(String targetDirectory) {
- this.targetDirectory = targetDirectory;
- }
-
- @Override
- public File getTargetDirectory() {
- return new File(targetDirectory);
- }
-
- @Override
- public Iterator<URL> getSource() {
- if (null != sourceDirectory) {
- return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator();
- }
- List<URL> sourceUrls = new ArrayList<URL>();
- if ( sourcePaths != null && sourcePaths.size() > 0) {
- for (String source : sourcePaths) {
- sourceUrls.add(URLUtil.parseURL(source));
- }
- }
- return sourceUrls.iterator();
- }
-
- public void setMaxDepth(int maxDepth) {
- this.maxDepth = maxDepth;
- }
-}
diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java
deleted file mode 100644
index a719e33..0000000
--- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java
+++ /dev/null
@@ -1,351 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.hive;
-
-import org.apache.streams.util.schema.FieldType;
-import org.apache.streams.util.schema.FieldUtil;
-import org.apache.streams.util.schema.FileUtil;
-import org.apache.streams.util.schema.Schema;
-import org.apache.streams.util.schema.SchemaStore;
-import org.apache.streams.util.schema.SchemaStoreImpl;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang3.StringUtils;
-import org.jsonschema2pojo.util.URLUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.apache.streams.util.schema.FileUtil.dropExtension;
-import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
-import static org.apache.streams.util.schema.FileUtil.swapExtension;
-import static org.apache.streams.util.schema.FileUtil.writeFile;
-
-/**
- * Generates hive table definitions for using org.openx.data.jsonserde.JsonSerDe on new-line delimited json documents.
- *
- *
- */
-public class StreamsHiveResourceGenerator implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHiveResourceGenerator.class);
-
- private static final String LS = System.getProperty("line.separator");
-
- private StreamsHiveGenerationConfig config;
-
- private SchemaStore schemaStore = new SchemaStoreImpl();
-
- private int currentDepth = 0;
-
- /**
- * Run from CLI without Maven
- *
- * <p/>
- * java -jar streams-plugin-hive-jar-with-dependencies.jar StreamsHiveResourceGenerator src/main/jsonschema target/generated-resources
- *
- * @param args [sourceDirectory, targetDirectory]
- * */
- public static void main(String[] args) {
- StreamsHiveGenerationConfig config = new StreamsHiveGenerationConfig();
-
- String sourceDirectory = "src/main/jsonschema";
- String targetDirectory = "target/generated-resources/hive";
-
- if ( args.length > 0 ) {
- sourceDirectory = args[0];
- }
- if ( args.length > 1 ) {
- targetDirectory = args[1];
- }
-
- config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory(targetDirectory);
-
- StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config);
- streamsHiveResourceGenerator.run();
- }
-
- public StreamsHiveResourceGenerator(StreamsHiveGenerationConfig config) {
- this.config = config;
- }
-
- @Override
- public void run() {
-
- Objects.requireNonNull(config);
-
- generate(config);
-
- }
-
- /**
- * run generate using supplied StreamsHiveGenerationConfig.
- * @param config StreamsHiveGenerationConfig
- */
- public void generate(StreamsHiveGenerationConfig config) {
-
- LinkedList<File> sourceFiles = new LinkedList<>();
-
- for (Iterator<URL> sources = config.getSource(); sources.hasNext();) {
- URL source = sources.next();
- sourceFiles.add(URLUtil.getFileFromURL(source));
- }
-
- LOGGER.info("Seeded with {} source paths:", sourceFiles.size());
-
- FileUtil.resolveRecursive(config, sourceFiles);
-
- LOGGER.info("Resolved {} schema files:", sourceFiles.size());
-
- for (File item : sourceFiles) {
- schemaStore.create(item.toURI());
- }
-
- LOGGER.info("Identified {} objects:", schemaStore.getSize());
-
- for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) {
- Schema schema = schemaIterator.next();
- currentDepth = 0;
- if ( schema.getUri().getScheme().equals("file")) {
- String inputFile = schema.getUri().getPath();
- String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
- for (String sourcePath : config.getSourcePaths()) {
- resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
- }
- String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "hql");
-
- LOGGER.info("Processing {}:", resourcePath);
-
- String resourceId = dropExtension(resourcePath).replace("/", "_");
-
- String resourceContent = generateResource(schema, resourceId);
-
- writeFile(outputFile, resourceContent);
-
- LOGGER.info("Wrote {}:", outputFile);
- }
- }
- }
-
- /**
- * generateResource String from schema and resourceId.
- * @param schema Schema
- * @param resourceId String
- * @return CREATE TABLE ...
- */
- public String generateResource(Schema schema, String resourceId) {
- StringBuilder resourceBuilder = new StringBuilder();
- resourceBuilder.append("CREATE TABLE ");
- resourceBuilder.append(hqlEscape(resourceId));
- resourceBuilder.append(LS);
- resourceBuilder.append("(");
- resourceBuilder.append(LS);
- resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId, ' ');
- resourceBuilder.append(")");
- resourceBuilder.append(LS);
- resourceBuilder.append("ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'");
- resourceBuilder.append(LS);
- resourceBuilder.append("WITH SERDEPROPERTIES (\"ignore.malformed.json\" = \"true\"");
- resourceBuilder.append(LS);
- resourceBuilder.append("STORED AS TEXTFILE");
- resourceBuilder.append(LS);
- resourceBuilder.append("LOCATION '${hiveconf:path}';");
- resourceBuilder.append(LS);
- return resourceBuilder.toString();
- }
-
- protected StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId, Character seperator) {
- ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId);
- if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) {
- builder = appendPropertiesNode(builder, schema, propertiesNode, seperator);
- }
- return builder;
- }
-
- private StringBuilder appendValueField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- builder.append(hqlEscape(fieldId));
- builder.append(seperator);
- builder.append(hqlType(fieldType));
- return builder;
- }
-
- protected StringBuilder appendArrayItems(StringBuilder builder, Schema schema, String fieldId, ObjectNode itemsNode, Character seperator) {
- // not safe to append nothing
- Objects.requireNonNull(builder);
- if ( itemsNode == null ) {
- return builder;
- }
- if ( itemsNode.has("type")) {
- try {
- FieldType itemType = FieldUtil.determineFieldType(itemsNode);
- switch ( itemType ) {
- case OBJECT:
- builder = appendArrayObject(builder, schema, fieldId, itemsNode, seperator);
- break;
- case ARRAY:
- ObjectNode subArrayItems = (ObjectNode) itemsNode.get("items");
- builder = appendArrayItems(builder, schema, fieldId, subArrayItems, seperator);
- break;
- default:
- builder = appendArrayField(builder, schema, fieldId, itemType, seperator);
- }
- } catch (Exception ex) {
- LOGGER.warn("No item type resolvable for {}", fieldId);
- }
- }
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendArrayField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- Objects.requireNonNull(fieldId);
- builder.append(hqlEscape(fieldId));
- builder.append(seperator);
- builder.append("ARRAY<" + hqlType(fieldType) + ">");
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendArrayObject(StringBuilder builder, Schema schema, String fieldId, ObjectNode fieldNode, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- Objects.requireNonNull(fieldNode);
- if (StringUtils.isNotBlank(fieldId)) {
- builder.append(hqlEscape(fieldId));
- builder.append(seperator);
- }
- builder.append("ARRAY");
- builder.append(LS);
- builder.append("<");
- builder.append(LS);
- ObjectNode propertiesNode = schemaStore.resolveProperties(schema, fieldNode, fieldId);
- builder = appendStructField(builder, schema, "", propertiesNode, ':');
- builder.append(">");
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendStructField(StringBuilder builder, Schema schema, String fieldId, ObjectNode propertiesNode, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- Objects.requireNonNull(propertiesNode);
-
- if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) {
-
- currentDepth += 1;
-
- if (StringUtils.isNotBlank(fieldId)) {
- builder.append(hqlEscape(fieldId));
- builder.append(seperator);
- }
- builder.append("STRUCT");
- builder.append(LS);
- builder.append("<");
- builder.append(LS);
-
- builder = appendPropertiesNode(builder, schema, propertiesNode, ':');
-
- builder.append(">");
- builder.append(LS);
-
- currentDepth -= 1;
-
- }
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendPropertiesNode(StringBuilder builder, Schema schema, ObjectNode propertiesNode, Character seperator) {
- Objects.requireNonNull(builder);
- Objects.requireNonNull(propertiesNode);
- Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields();
- List<String> fieldStrings = new ArrayList<>();
- for ( ; fields.hasNext(); ) {
- Map.Entry<String, JsonNode> field = fields.next();
- String fieldId = field.getKey();
- if ( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) {
- ObjectNode fieldNode = (ObjectNode) field.getValue();
- FieldType fieldType = FieldUtil.determineFieldType(fieldNode);
- if (fieldType != null ) {
- switch (fieldType) {
- case ARRAY:
- ObjectNode itemsNode = (ObjectNode) fieldNode.get("items");
- if ( currentDepth <= config.getMaxDepth()) {
- StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, itemsNode, seperator);
- if (StringUtils.isNotBlank(arrayItemsBuilder.toString())) {
- fieldStrings.add(arrayItemsBuilder.toString());
- }
- }
- break;
- case OBJECT:
- ObjectNode childProperties = schemaStore.resolveProperties(schema, fieldNode, fieldId);
- if ( currentDepth < config.getMaxDepth()) {
- StringBuilder structFieldBuilder = appendStructField(new StringBuilder(), schema, fieldId, childProperties, seperator);
- if (StringUtils.isNotBlank(structFieldBuilder.toString())) {
- fieldStrings.add(structFieldBuilder.toString());
- }
- }
- break;
- default:
- StringBuilder valueFieldBuilder = appendValueField(new StringBuilder(), schema, fieldId, fieldType, seperator);
- if (StringUtils.isNotBlank(valueFieldBuilder.toString())) {
- fieldStrings.add(valueFieldBuilder.toString());
- }
- }
- }
- }
- }
- builder.append(String.join("," + LS, fieldStrings)).append(LS);
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private static String hqlEscape( String fieldId ) {
- return "`" + fieldId + "`";
- }
-
- private static String hqlType( FieldType fieldType ) {
- switch ( fieldType ) {
- case INTEGER:
- return "INT";
- case NUMBER:
- return "FLOAT";
- case OBJECT:
- return "STRUCT";
- default:
- return fieldType.name().toUpperCase();
- }
- }
-
-}
diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java
deleted file mode 100644
index eb9490d..0000000
--- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.hive;
-
-import org.apache.maven.plugin.AbstractMojo;
-import org.apache.maven.plugin.MojoExecutionException;
-import org.apache.maven.plugin.MojoFailureException;
-import org.apache.maven.plugins.annotations.Component;
-import org.apache.maven.plugins.annotations.Execute;
-import org.apache.maven.plugins.annotations.LifecyclePhase;
-import org.apache.maven.plugins.annotations.Mojo;
-import org.apache.maven.plugins.annotations.Parameter;
-import org.apache.maven.project.MavenProject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.List;
-
-@Mojo (
- name = "generate-resources",
- defaultPhase = LifecyclePhase.GENERATE_RESOURCES
- )
-@Execute (
- goal = "generate-resources",
- phase = LifecyclePhase.GENERATE_RESOURCES
- )
-public class StreamsHiveResourceGeneratorMojo extends AbstractMojo {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHiveResourceGeneratorMojo.class);
-
- private volatile MojoFailureException mojoFailureException;
-
- @Component
- private MavenProject project;
-
- @Parameter( defaultValue = "${project.basedir}", readonly = true )
- private File basedir;
-
- @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only
- public String sourceDirectory;
-
- @Parameter( readonly = true ) // Maven 3 only
- public List<String> sourcePaths;
-
- @Parameter(defaultValue = "./target/generated-resources/hive", readonly = true)
- public String targetDirectory;
-
- /**
- * execute StreamsHiveResourceGeneratorMojo.
- * @throws MojoExecutionException MojoExecutionException
- * @throws MojoFailureException MojoFailureException
- */
- public void execute() throws MojoExecutionException, MojoFailureException {
-
- //addProjectDependenciesToClasspath();
-
- StreamsHiveGenerationConfig config = new StreamsHiveGenerationConfig();
-
- if ( sourcePaths != null && sourcePaths.size() > 0) {
- config.setSourcePaths(sourcePaths);
- } else {
- config.setSourceDirectory(sourceDirectory);
- }
- config.setTargetDirectory(targetDirectory);
-
- StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config);
-
- streamsHiveResourceGenerator.run();
-
- }
-
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hive/src/site/markdown/index.md b/streams-plugins/streams-plugin-hive/src/site/markdown/index.md
deleted file mode 100644
index 5a873c3..0000000
--- a/streams-plugins/streams-plugin-hive/src/site/markdown/index.md
+++ /dev/null
@@ -1,40 +0,0 @@
-org.apache.streams.plugins:streams-plugin-hive
-==============================================
-
-streams-plugin-hive generates resources from json schemas to assist with analysis of json data using Apache Hive.
-
-### Usage
-
-Output will be placed in target/generated-resources/hive by default
-
-##### Maven
-
-Run within a module containing a src/main/jsonschema directory
-
- mvn org.apache.streams.plugins:streams-plugin-hive:0.4-incubating:generate-resources
-
-[streams-plugin-hive/pom.xml](streams-plugin-hive/pom.xml "streams-plugin-hive/pom.xml")
-
-##### SDK
-
-Embed within your own java code
-
- StreamsHiveGenerationConfig config = new StreamsHiveGenerationConfig();
- config.setSourceDirectory("src/main/jsonschema");
- config.setTargetDirectory("target/generated-resources");
- StreamsHiveGenerationConfig generator = new StreamsHiveGenerationConfig(config);
- generator.run();
-
-##### CLI
-
-Run from CLI without Maven
-
- java -jar streams-plugin-hive-jar-with-dependencies.jar StreamsHiveResourceGenerator src/main/jsonschema target/generated-resources
-
-#### Example
-
-#### Documentation
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java
deleted file mode 100644
index 7031047..0000000
--- a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.test;
-
-import org.apache.streams.plugins.hive.StreamsHiveResourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Test;
-import org.testng.Assert;
-
-import java.io.File;
-import java.util.Collection;
-
-/**
- * Test whether StreamsHiveResourceGeneratorCLI generates resources.
- */
-public class StreamsHiveResourceGeneratorCLITest {
-
- @Test
- public void testStreamsHiveResourceGeneratorCLI() throws Exception {
-
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
- String targetDirectory = "target/generated-resources/hive-cli";
-
- StreamsHiveResourceGenerator.main(new String[]{sourceDirectory, targetDirectory});
-
- File testOutput = new File(targetDirectory);
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> testOutputFiles = FileUtils.listFiles(testOutput, StreamsHiveResourceGeneratorTest.hqlFilter, true);
- Assert.assertEquals(testOutputFiles.size(), 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoIT.java b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoIT.java
deleted file mode 100644
index f174e28..0000000
--- a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoIT.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.test;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.maven.it.Verifier;
-import org.apache.maven.it.util.ResourceExtractor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests that streams-plugin-hive running via maven generates txt resources.
- */
-public class StreamsHiveResourceGeneratorMojoIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHiveResourceGeneratorMojoIT.class);
-
- @Test
- public void testStreamsHiveResourceGeneratorMojo() throws Exception {
-
- File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-hive" );
-
- Verifier verifier;
-
- verifier = new Verifier( testDir.getAbsolutePath() );
-
- List<String> cliOptions = new ArrayList<>();
- cliOptions.add( "-N" );
- verifier.executeGoals(Stream.of(
- "clean",
- "dependency:unpack-dependencies",
- "generate-resources").collect(Collectors.toList()));
-
- verifier.verifyErrorFreeLog();
-
- verifier.resetStreams();
-
- File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/hive-mojo");
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, StreamsHiveResourceGeneratorTest.hqlFilter, true);
- Assert.assertEquals (outputCollection.size(), 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java
deleted file mode 100644
index 9bd6107..0000000
--- a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.test;
-
-import org.apache.streams.plugins.hive.StreamsHiveGenerationConfig;
-import org.apache.streams.plugins.hive.StreamsHiveResourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.Collection;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Test that Activity beans are compatible with the example activities in the spec.
- */
-@Ignore
-public class StreamsHiveResourceGeneratorTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHiveResourceGeneratorTest.class);
-
- public static final String[] hqlFilter = new String[]{"hql"};
-
- /**
- * Tests that all example activities can be loaded into Activity beans.
- *
- * @throws Exception Exception
- */
- @Test
- public void testStreamsHiveResourceGenerator() throws Exception {
-
- StreamsHiveGenerationConfig config = new StreamsHiveGenerationConfig();
-
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
-
- config.setSourceDirectory(sourceDirectory);
-
- config.setTargetDirectory("target/generated-resources/test");
-
- config.setExclusions(Stream.of("attachments").collect(Collectors.toSet()));
-
- config.setMaxDepth(2);
-
- StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config);
- streamsHiveResourceGenerator.run();
-
- File testOutput = config.getTargetDirectory();
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> testOutputFiles = FileUtils.listFiles(testOutput, hqlFilter, true);
- Assert.assertEquals(testOutputFiles.size(), 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/Tweet.hql b/streams-plugins/streams-plugin-hive/src/test/resources/Tweet.hql
deleted file mode 100644
index 4cec1ac..0000000
--- a/streams-plugins/streams-plugin-hive/src/test/resources/Tweet.hql
+++ /dev/null
@@ -1,313 +0,0 @@
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
-/*
-root
- |-- contributors: string (nullable = true)
- |-- coordinates: string (nullable = true)
- |-- created_at: string (nullable = true)
- |-- entities: struct (nullable = true)
- | |-- hashtags: array (nullable = true)
- | | |-- element: struct (containsNull = true)
- | | | |-- indices: array (nullable = true)
- | | | | |-- element: long (containsNull = true)
- | | | |-- text: string (nullable = true)
- | |-- symbols: array (nullable = true)
- | | |-- element: string (containsNull = true)
- | |-- urls: array (nullable = true)
- | | |-- element: struct (containsNull = true)
- | | | |-- display_url: string (nullable = true)
- | | | |-- expanded_url: string (nullable = true)
- | | | |-- indices: array (nullable = true)
- | | | | |-- element: long (containsNull = true)
- | | | |-- url: string (nullable = true)
- | |-- user_mentions: array (nullable = true)
- | | |-- element: struct (containsNull = true)
- | | | |-- id: long (nullable = true)
- | | | |-- id_str: string (nullable = true)
- | | | |-- indices: array (nullable = true)
- | | | | |-- element: long (containsNull = true)
- | | | |-- name: string (nullable = true)
- | | | |-- screen_name: string (nullable = true)
- |-- favorite_count: long (nullable = true)
- |-- favorited: boolean (nullable = true)
- |-- geo: string (nullable = true)
- |-- id: long (nullable = true)
- |-- id_str: string (nullable = true)
- |-- in_reply_to_screen_name: string (nullable = true)
- |-- in_reply_to_status_id: long (nullable = true)
- |-- in_reply_to_status_id_str: string (nullable = true)
- |-- in_reply_to_user_id: long (nullable = true)
- |-- in_reply_to_user_id_str: string (nullable = true)
- |-- lang: string (nullable = true)
- |-- place: struct (nullable = true)
- | |-- attributes: struct (nullable = true)
- | |-- bounding_box: struct (nullable = true)
- | | |-- coordinates: array (nullable = true)
- | | | |-- element: array (containsNull = true)
- | | | | |-- element: array (containsNull = true)
- | | | | | |-- element: double (containsNull = true)
- | | |-- type: string (nullable = true)
- | |-- contained_within: array (nullable = true)
- | | |-- element: string (containsNull = true)
- | |-- country: string (nullable = true)
- | |-- country_code: string (nullable = true)
- | |-- full_name: string (nullable = true)
- | |-- id: string (nullable = true)
- | |-- name: string (nullable = true)
- | |-- place_type: string (nullable = true)
- | |-- url: string (nullable = true)
- |-- possibly_sensitive: boolean (nullable = true)
- |-- retweet_count: long (nullable = true)
- |-- retweeted: boolean (nullable = true)
- |-- retweeted_status: struct (nullable = true)
- | |-- contributors: string (nullable = true)
- | |-- coordinates: string (nullable = true)
- | |-- created_at: string (nullable = true)
- | |-- entities: struct (nullable = true)
- | | |-- hashtags: array (nullable = true)
- | | | |-- element: struct (containsNull = true)
- | | | | |-- indices: array (nullable = true)
- | | | | | |-- element: long (containsNull = true)
- | | | | |-- text: string (nullable = true)
- | | |-- symbols: array (nullable = true)
- | | | |-- element: string (containsNull = true)
- | | |-- urls: array (nullable = true)
- | | | |-- element: struct (containsNull = true)
- | | | | |-- display_url: string (nullable = true)
- | | | | |-- expanded_url: string (nullable = true)
- | | | | |-- indices: array (nullable = true)
- | | | | | |-- element: long (containsNull = true)
- | | | | |-- url: string (nullable = true)
- | | |-- user_mentions: array (nullable = true)
- | | | |-- element: struct (containsNull = true)
- | | | | |-- id: long (nullable = true)
- | | | | |-- id_str: string (nullable = true)
- | | | | |-- indices: array (nullable = true)
- | | | | | |-- element: long (containsNull = true)
- | | | | |-- name: string (nullable = true)
- | | | | |-- screen_name: string (nullable = true)
- | |-- favorite_count: long (nullable = true)
- | |-- favorited: boolean (nullable = true)
- | |-- geo: string (nullable = true)
- | |-- id: long (nullable = true)
- | |-- id_str: string (nullable = true)
- | |-- in_reply_to_screen_name: string (nullable = true)
- | |-- in_reply_to_status_id: string (nullable = true)
- | |-- in_reply_to_status_id_str: string (nullable = true)
- | |-- in_reply_to_user_id: string (nullable = true)
- | |-- in_reply_to_user_id_str: string (nullable = true)
- | |-- lang: string (nullable = true)
- | |-- place: struct (nullable = true)
- | | |-- attributes: struct (nullable = true)
- | | |-- bounding_box: struct (nullable = true)
- | | | |-- coordinates: array (nullable = true)
- | | | | |-- element: array (containsNull = true)
- | | | | | |-- element: array (containsNull = true)
- | | | | | | |-- element: double (containsNull = true)
- | | | |-- type: string (nullable = true)
- | | |-- contained_within: array (nullable = true)
- | | | |-- element: string (containsNull = true)
- | | |-- country: string (nullable = true)
- | | |-- country_code: string (nullable = true)
- | | |-- full_name: string (nullable = true)
- | | |-- id: string (nullable = true)
- | | |-- name: string (nullable = true)
- | | |-- place_type: string (nullable = true)
- | | |-- url: string (nullable = true)
- | |-- possibly_sensitive: boolean (nullable = true)
- | |-- retweet_count: long (nullable = true)
- | |-- retweeted: boolean (nullable = true)
- | |-- source: string (nullable = true)
- | |-- text: string (nullable = true)
- | |-- truncated: boolean (nullable = true)
- | |-- user: struct (nullable = true)
- | | |-- contributors_enabled: boolean (nullable = true)
- | | |-- created_at: string (nullable = true)
- | | |-- default_profile: boolean (nullable = true)
- | | |-- default_profile_image: boolean (nullable = true)
- | | |-- description: string (nullable = true)
- | | |-- entities: struct (nullable = true)
- | | | |-- description: struct (nullable = true)
- | | | | |-- urls: array (nullable = true)
- | | | | | |-- element: struct (containsNull = true)
- | | | | | | |-- display_url: string (nullable = true)
- | | | | | | |-- expanded_url: string (nullable = true)
- | | | | | | |-- indices: array (nullable = true)
- | | | | | | | |-- element: long (containsNull = true)
- | | | | | | |-- url: string (nullable = true)
- | | | |-- url: struct (nullable = true)
- | | | | |-- urls: array (nullable = true)
- | | | | | |-- element: struct (containsNull = true)
- | | | | | | |-- display_url: string (nullable = true)
- | | | | | | |-- expanded_url: string (nullable = true)
- | | | | | | |-- indices: array (nullable = true)
- | | | | | | | |-- element: long (containsNull = true)
- | | | | | | |-- url: string (nullable = true)
- | | |-- favourites_count: long (nullable = true)
- | | |-- follow_request_sent: boolean (nullable = true)
- | | |-- followers_count: long (nullable = true)
- | | |-- following: boolean (nullable = true)
- | | |-- friends_count: long (nullable = true)
- | | |-- geo_enabled: boolean (nullable = true)
- | | |-- id: long (nullable = true)
- | | |-- id_str: string (nullable = true)
- | | |-- is_translation_enabled: boolean (nullable = true)
- | | |-- is_translator: boolean (nullable = true)
- | | |-- lang: string (nullable = true)
- | | |-- listed_count: long (nullable = true)
- | | |-- location: string (nullable = true)
- | | |-- name: string (nullable = true)
- | | |-- notifications: boolean (nullable = true)
- | | |-- profile_background_color: string (nullable = true)
- | | |-- profile_background_image_url: string (nullable = true)
- | | |-- profile_background_image_url_https: string (nullable = true)
- | | |-- profile_background_tile: boolean (nullable = true)
- | | |-- profile_banner_url: string (nullable = true)
- | | |-- profile_image_url: string (nullable = true)
- | | |-- profile_image_url_https: string (nullable = true)
- | | |-- profile_link_color: string (nullable = true)
- | | |-- profile_sidebar_border_color: string (nullable = true)
- | | |-- profile_sidebar_fill_color: string (nullable = true)
- | | |-- profile_text_color: string (nullable = true)
- | | |-- profile_use_background_image: boolean (nullable = true)
- | | |-- protected: boolean (nullable = true)
- | | |-- screen_name: string (nullable = true)
- | | |-- statuses_count: long (nullable = true)
- | | |-- time_zone: string (nullable = true)
- | | |-- url: string (nullable = true)
- | | |-- utc_offset: long (nullable = true)
- | | |-- verified: boolean (nullable = true)
- |-- source: string (nullable = true)
- |-- text: string (nullable = true)
- |-- truncated: boolean (nullable = true)
- |-- user: struct (nullable = true)
- | |-- contributors_enabled: boolean (nullable = true)
- | |-- created_at: string (nullable = true)
- | |-- default_profile: boolean (nullable = true)
- | |-- default_profile_image: boolean (nullable = true)
- | |-- description: string (nullable = true)
- | |-- entities: struct (nullable = true)
- | | |-- description: struct (nullable = true)
- | | | |-- urls: array (nullable = true)
- | | | | |-- element: string (containsNull = true)
- | | |-- url: struct (nullable = true)
- | | | |-- urls: array (nullable = true)
- | | | | |-- element: struct (containsNull = true)
- | | | | | |-- display_url: string (nullable = true)
- | | | | | |-- expanded_url: string (nullable = true)
- | | | | | |-- indices: array (nullable = true)
- | | | | | | |-- element: long (containsNull = true)
- | | | | | |-- url: string (nullable = true)
- | |-- favourites_count: long (nullable = true)
- | |-- follow_request_sent: boolean (nullable = true)
- | |-- followers_count: long (nullable = true)
- | |-- following: boolean (nullable = true)
- | |-- friends_count: long (nullable = true)
- | |-- geo_enabled: boolean (nullable = true)
- | |-- id: long (nullable = true)
- | |-- id_str: string (nullable = true)
- | |-- is_translation_enabled: boolean (nullable = true)
- | |-- is_translator: boolean (nullable = true)
- | |-- lang: string (nullable = true)
- | |-- listed_count: long (nullable = true)
- | |-- location: string (nullable = true)
- | |-- name: string (nullable = true)
- | |-- notifications: boolean (nullable = true)
- | |-- profile_background_color: string (nullable = true)
- | |-- profile_background_image_url: string (nullable = true)
- | |-- profile_background_image_url_https: string (nullable = true)
- | |-- profile_background_tile: boolean (nullable = true)
- | |-- profile_banner_url: string (nullable = true)
- | |-- profile_image_url: string (nullable = true)
- | |-- profile_image_url_https: string (nullable = true)
- | |-- profile_link_color: string (nullable = true)
- | |-- profile_sidebar_border_color: string (nullable = true)
- | |-- profile_sidebar_fill_color: string (nullable = true)
- | |-- profile_text_color: string (nullable = true)
- | |-- profile_use_background_image: boolean (nullable = true)
- | |-- protected: boolean (nullable = true)
- | |-- screen_name: string (nullable = true)
- | |-- statuses_count: long (nullable = true)
- | |-- time_zone: string (nullable = true)
- | |-- url: string (nullable = true)
- | |-- utc_offset: long (nullable = true)
- | |-- verified: boolean (nullable = true)
-*/
-create table tweet (
- created_at string,
- entities struct
- <
- hashtags: array
- <
- struct
- <
- text: string
- >
- >,
- urls: array
- <
- struct
- <
- expanded_url: string
- >
- >,
- user_mentions: array
- <
- struct
- <
- id_str: string
- >
- >
- >,
- in_reply_to_user_id_str: string,
- user: struct
- <
- id_str: string
- >,
- retweeted_status struct
- <
- entities struct
- <
- hashtags: array
- <
- struct
- <
- text: string
- >
- >,
- urls: array
- <
- struct
- <
- expanded_url: string
- >
- >,
- user_mentions: array
- <
- struct
- <
- id_str: string
- >
- >
- >,
- in_reply_to_user_id_str: string,
- user: struct
- <
- id_str: string
- >
- >
-)
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/expected/activity.hql b/streams-plugins/streams-plugin-hive/src/test/resources/expected/activity.hql
deleted file mode 100644
index 9c7638c..0000000
--- a/streams-plugins/streams-plugin-hive/src/test/resources/expected/activity.hql
+++ /dev/null
@@ -1,219 +0,0 @@
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
-CREATE TABLE `activity`
-(
-`id` STRING,
-`actor` STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`verb` STRING,
-`object` STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`target` STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published` STRING,
-`updated` STRING,
-`generator` STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`icon` STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`provider` STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`title` STRING,
-`content` STRING,
-`url` STRING,
-`links` ARRAY<STRING>
-)
-ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
-WITH SERDEPROPERTIES ("ignore.malformed.json" = "true"
-STORED AS TEXTFILE
-LOCATION '${hiveconf:path}';
diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/expected/collection.hql b/streams-plugins/streams-plugin-hive/src/test/resources/expected/collection.hql
deleted file mode 100644
index 01fcb4a..0000000
--- a/streams-plugins/streams-plugin-hive/src/test/resources/expected/collection.hql
+++ /dev/null
@@ -1,63 +0,0 @@
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
-CREATE TABLE `collection`
-(
-`url` STRING,
-`totalItems` INT,
-`items` ARRAY
-<
-STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
->
-)
-ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
-WITH SERDEPROPERTIES ("ignore.malformed.json" = "true"
-STORED AS TEXTFILE
-LOCATION '${hiveconf:path}';
diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/expected/media_link.hql b/streams-plugins/streams-plugin-hive/src/test/resources/expected/media_link.hql
deleted file mode 100644
index aa604ec..0000000
--- a/streams-plugins/streams-plugin-hive/src/test/resources/expected/media_link.hql
+++ /dev/null
@@ -1,27 +0,0 @@
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
-CREATE TABLE `media_link`
-(
-`duration` FLOAT,
-`height` INT,
-`width` INT,
-`url` STRING
-)
-ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
-WITH SERDEPROPERTIES ("ignore.malformed.json" = "true"
-STORED AS TEXTFILE
-LOCATION '${hiveconf:path}';
diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/expected/object.hql b/streams-plugins/streams-plugin-hive/src/test/resources/expected/object.hql
deleted file mode 100644
index 72ccc82..0000000
--- a/streams-plugins/streams-plugin-hive/src/test/resources/expected/object.hql
+++ /dev/null
@@ -1,77 +0,0 @@
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
-CREATE TABLE `object`
-(
-`id` STRING,
-`image` STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName` STRING,
-`summary` STRING,
-`content` STRING,
-`url` STRING,
-`objectType` STRING,
-`author` STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published` STRING,
-`updated` STRING,
-`upstreamDuplicates` ARRAY<STRING>,
-`downstreamDuplicates` ARRAY<STRING>
-)
-ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
-WITH SERDEPROPERTIES ("ignore.malformed.json" = "true"
-STORED AS TEXTFILE
-LOCATION '${hiveconf:path}';
diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/expected/objectTypes/place.hql b/streams-plugins/streams-plugin-hive/src/test/resources/expected/objectTypes/place.hql
deleted file mode 100644
index 3846304..0000000
--- a/streams-plugins/streams-plugin-hive/src/test/resources/expected/objectTypes/place.hql
+++ /dev/null
@@ -1,96 +0,0 @@
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
-
-CREATE TABLE `objectTypes_place`
-(
-`id` STRING,
-`image` STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName` STRING,
-`summary` STRING,
-`content` STRING,
-`url` STRING,
-`objectType` STRING,
-`author` STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published` STRING,
-`updated` STRING,
-`upstreamDuplicates` ARRAY<STRING>,
-`downstreamDuplicates` ARRAY<STRING>,
-`address` STRUCT
-<
-`post-office-box`:STRING,
-`extended-address`:STRING,
-`street-address`:STRING,
-`locality`:STRING,
-`region`:STRING,
-`postal-code`:STRING,
-`country-name`:STRING
->
-,
-`position` STRUCT
-<
-`altitude`:FLOAT,
-`latitude`:FLOAT,
-`longitude`:FLOAT
->
-
-)
-ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
-WITH SERDEPROPERTIES ("ignore.malformed.json" = "true"
-STORED AS TEXTFILE
-LOCATION '${hiveconf:path}';
diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/expected/verbs/purchase.hql b/streams-plugins/streams-plugin-hive/src/test/resources/expected/verbs/purchase.hql
deleted file mode 100644
index 3f3db77..0000000
--- a/streams-plugins/streams-plugin-hive/src/test/resources/expected/verbs/purchase.hql
+++ /dev/null
@@ -1,220 +0,0 @@
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
-
-CREATE TABLE `verbs_purchase`
-(
-`id` STRING,
-`actor` STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`verb` STRING,
-`object` STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`target` STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published` STRING,
-`updated` STRING,
-`generator` STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`icon` STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`provider` STRUCT
-<
-`id`:STRING,
-`image`:STRUCT
-<
-`duration`:FLOAT,
-`height`:INT,
-`width`:INT,
-`url`:STRING
->
-,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`author`:STRUCT
-<
-`id`:STRING,
-`displayName`:STRING,
-`summary`:STRING,
-`content`:STRING,
-`url`:STRING,
-`objectType`:STRING,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`published`:STRING,
-`updated`:STRING,
-`upstreamDuplicates`:ARRAY<STRING>,
-`downstreamDuplicates`:ARRAY<STRING>
->
-,
-`title` STRING,
-`content` STRING,
-`url` STRING,
-`links` ARRAY<STRING>
-)
-ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
-WITH SERDEPROPERTIES ("ignore.malformed.json" = "true"
-STORED AS TEXTFILE
-LOCATION '${hiveconf:path}';
diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml b/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml
deleted file mode 100644
index 5e338e7..0000000
--- a/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml
+++ /dev/null
@@ -1,144 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-hive-test</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Test StreamsPojoHiveMojo</name>
-
- <repositories>
- <repository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- </repositories>
-
- <pluginRepositories>
- <pluginRepository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </pluginRepository>
- <pluginRepository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </pluginRepository>
- </pluginRepositories>
-
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-hive</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <configuration>
- <sourcePaths>
- <sourcePath>target/test-classes/activitystreams-schemas/activity.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/collection.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/media_link.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/object.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/objectTypes</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/verbs</sourcePath>
- </sourcePaths>
- <targetDirectory>target/generated-resources/hive-mojo</targetDirectory>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate-resources</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- </configuration>
- <executions>
- <execution>
- <id>unpack-schemas</id>
- <phase>process-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pig/pom.xml b/streams-plugins/streams-plugin-pig/pom.xml
deleted file mode 100644
index aa4f317..0000000
--- a/streams-plugins/streams-plugin-pig/pom.xml
+++ /dev/null
@@ -1,196 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
- <artifactId>streams-plugin-pig</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <packaging>maven-plugin</packaging>
-
- <parent>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugins</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <properties>
- <pig.version>1.0</pig.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>commons-logging</artifactId>
- <groupId>commons-logging</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-aether-provider</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-artifact</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-compat</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-plugin-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.shared</groupId>
- <artifactId>maven-verifier</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.codehaus.plexus</groupId>
- <artifactId>plexus-utils</artifactId>
- <version>3.0.15</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-plugin-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-generators</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-testing</groupId>
- <artifactId>maven-plugin-testing-harness</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>org.joda</groupId>
- <artifactId>joda-convert</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <artifactId>maven-plugin-plugin</artifactId>
- <version>3.4</version>
- <configuration>
- <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
- </configuration>
- <executions>
- <execution>
- <id>mojo-descriptor</id>
- <phase>process-classes</phase>
- <goals>
- <goal>descriptor</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>resource-dependencies</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includes>**/*.json</includes>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
-</project>
diff --git a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java
deleted file mode 100644
index c4358a0..0000000
--- a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.pig;
-
-import org.apache.streams.util.schema.GenerationConfig;
-
-import org.jsonschema2pojo.DefaultGenerationConfig;
-import org.jsonschema2pojo.util.URLUtil;
-
-import java.io.File;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Configures StreamsPigResourceGenerator.
- */
-public class StreamsPigGenerationConfig extends DefaultGenerationConfig implements GenerationConfig {
-
- public String getSourceDirectory() {
- return sourceDirectory;
- }
-
- public List<String> getSourcePaths() {
- return sourcePaths;
- }
-
- private String sourceDirectory;
- private List<String> sourcePaths = new ArrayList<String>();
- private String targetDirectory;
- private int maxDepth = 1;
-
- public Set<String> getExclusions() {
- return exclusions;
- }
-
- public void setExclusions(Set<String> exclusions) {
- this.exclusions = exclusions;
- }
-
- private Set<String> exclusions = new HashSet<String>();
-
- public int getMaxDepth() {
- return maxDepth;
- }
-
- public void setSourceDirectory(String sourceDirectory) {
- this.sourceDirectory = sourceDirectory;
- }
-
- public void setSourcePaths(List<String> sourcePaths) {
- this.sourcePaths = sourcePaths;
- }
-
- public void setTargetDirectory(String targetDirectory) {
- this.targetDirectory = targetDirectory;
- }
-
- @Override
- public File getTargetDirectory() {
- return new File(targetDirectory);
- }
-
- /**
- * get all sources.
- * @return Iterator of URL
- */
- @Override
- public Iterator<URL> getSource() {
- if (null != sourceDirectory) {
- return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator();
- }
- List<URL> sourceUrls = new ArrayList<URL>();
- if ( sourcePaths != null && sourcePaths.size() > 0) {
- for (String source : sourcePaths) {
- sourceUrls.add(URLUtil.parseURL(source));
- }
- }
- return sourceUrls.iterator();
- }
-
- public void setMaxDepth(int maxDepth) {
- this.maxDepth = maxDepth;
- }
-}
diff --git a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java
deleted file mode 100644
index 2783f38..0000000
--- a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java
+++ /dev/null
@@ -1,356 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.pig;
-
-import org.apache.streams.util.schema.FieldType;
-import org.apache.streams.util.schema.FieldUtil;
-import org.apache.streams.util.schema.FileUtil;
-import org.apache.streams.util.schema.Schema;
-import org.apache.streams.util.schema.SchemaStore;
-import org.apache.streams.util.schema.SchemaStoreImpl;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang3.StringUtils;
-import org.jsonschema2pojo.util.URLUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.apache.streams.util.schema.FileUtil.dropExtension;
-import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
-import static org.apache.streams.util.schema.FileUtil.swapExtension;
-import static org.apache.streams.util.schema.FileUtil.writeFile;
-
-/**
- * Embed within your own java code
- *
- * <p/>
- * StreamsPigGenerationConfig config = new StreamsPigGenerationConfig();
- * config.setSourceDirectory("src/main/jsonschema");
- * config.setTargetDirectory("target/generated-resources");
- * StreamsPigResourceGenerator generator = new StreamsPigResourceGenerator(config);
- * generator.run();
- *
- */
-public class StreamsPigResourceGenerator implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsPigResourceGenerator.class);
-
- private static final String LS = System.getProperty("line.separator");
-
- private StreamsPigGenerationConfig config;
-
- private SchemaStore schemaStore = new SchemaStoreImpl();
-
- private int currentDepth = 0;
-
- /**
- * Run from CLI without Maven
- *
- * <p/>
- * java -jar streams-plugin-pig-jar-with-dependencies.jar StreamsPigResourceGenerator src/main/jsonschema target/generated-resources
- *
- * @param args [sourceDirectory, targetDirectory]
- * */
- public static void main(String[] args) {
- StreamsPigGenerationConfig config = new StreamsPigGenerationConfig();
-
- String sourceDirectory = "src/main/jsonschema";
- String targetDirectory = "target/generated-resources/pig-cli";
-
- if ( args.length > 0 ) {
- sourceDirectory = args[0];
- }
- if ( args.length > 1 ) {
- targetDirectory = args[1];
- }
-
- config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory(targetDirectory);
-
- StreamsPigResourceGenerator streamsPigResourceGenerator = new StreamsPigResourceGenerator(config);
- streamsPigResourceGenerator.run();
-
- }
-
- public StreamsPigResourceGenerator(StreamsPigGenerationConfig config) {
- this.config = config;
- }
-
- @Override
- public void run() {
-
- Objects.requireNonNull(config);
-
- generate(config);
-
- }
-
- /**
- * run generate using supplied StreamsPigGenerationConfig.
- * @param config StreamsPigGenerationConfig
- */
- public void generate(StreamsPigGenerationConfig config) {
-
- List<File> sourceFiles = new LinkedList<>();
-
- for (Iterator<URL> sources = config.getSource(); sources.hasNext(); ) {
- URL source = sources.next();
- sourceFiles.add(URLUtil.getFileFromURL(source));
- }
-
- LOGGER.info("Seeded with {} source paths:", sourceFiles.size());
-
- FileUtil.resolveRecursive(config, sourceFiles);
-
- LOGGER.info("Resolved {} schema files:", sourceFiles.size());
-
- for (File item : sourceFiles) {
- schemaStore.create(item.toURI());
- }
-
- LOGGER.info("Identified {} objects:", schemaStore.getSize());
-
- for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) {
- Schema schema = schemaIterator.next();
- currentDepth = 0;
- if (schema.getUri().getScheme().equals("file")) {
- String inputFile = schema.getUri().getPath();
- String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
- for (String sourcePath : config.getSourcePaths()) {
- resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
- }
- String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "pig");
-
- LOGGER.info("Processing {}:", resourcePath);
-
- String resourceId = schemaSymbol(schema);
-
- String resourceContent = generateResource(schema, resourceId);
-
- writeFile(outputFile, resourceContent);
-
- LOGGER.info("Wrote {}:", outputFile);
- }
- }
- }
-
- /**
- * generateResource String from schema and resourceId.
- * @param schema Schema
- * @param resourceId String
- * @return mapping
- */
- public String generateResource(Schema schema, String resourceId) {
- StringBuilder resourceBuilder = new StringBuilder();
- resourceBuilder.append(pigEscape(resourceId));
- resourceBuilder.append(" = ");
- resourceBuilder.append("LOAD '' USING JsonLoader('");
- resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId, ':');
- resourceBuilder.append("');");
- return resourceBuilder.toString();
- }
-
- protected StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId, Character separator) {
- ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId);
- if (propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) {
- builder = appendPropertiesNode(builder, schema, propertiesNode, separator);
- }
- return builder;
- }
-
- private StringBuilder appendPropertiesNode(StringBuilder builder, Schema schema, ObjectNode propertiesNode, Character seperator) {
- Objects.requireNonNull(builder);
- Objects.requireNonNull(propertiesNode);
- Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields();
- List<String> fieldStrings = new ArrayList<>();
- for ( ; fields.hasNext(); ) {
- Map.Entry<String, JsonNode> field = fields.next();
- String fieldId = field.getKey();
- if ( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) {
- ObjectNode fieldNode = (ObjectNode) field.getValue();
- FieldType fieldType = FieldUtil.determineFieldType(fieldNode);
- if (fieldType != null ) {
- switch (fieldType) {
- case ARRAY:
- ObjectNode resolvedItems = schemaStore.resolveItems(schema, fieldNode, fieldId);
- if ( resolvedItems != null && currentDepth <= config.getMaxDepth()) {
- StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, resolvedItems, seperator);
- if (StringUtils.isNotBlank(arrayItemsBuilder.toString())) {
- fieldStrings.add(arrayItemsBuilder.toString());
- }
- }
- break;
- case OBJECT:
- ObjectNode childProperties = schemaStore.resolveProperties(schema, fieldNode, fieldId);
- if ( currentDepth < config.getMaxDepth()) {
- StringBuilder structFieldBuilder = appendStructField(new StringBuilder(), schema, fieldId, childProperties, seperator);
- if (StringUtils.isNotBlank(structFieldBuilder.toString())) {
- fieldStrings.add(structFieldBuilder.toString());
- }
- }
- break;
- default:
- StringBuilder valueFieldBuilder = appendValueField(new StringBuilder(), schema, fieldId, fieldType, seperator);
- if (StringUtils.isNotBlank(valueFieldBuilder.toString())) {
- fieldStrings.add(valueFieldBuilder.toString());
- }
- }
- }
- }
- }
- builder.append(String.join(", ", fieldStrings));
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendValueField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- builder.append(pigEscape(fieldId));
- builder.append(seperator);
- builder.append(pigType(fieldType));
- return builder;
- }
-
- protected StringBuilder appendArrayItems(StringBuilder builder, Schema schema, String fieldId, ObjectNode itemsNode, Character seperator) {
- // not safe to append nothing
- Objects.requireNonNull(builder);
- if ( itemsNode == null ) {
- return builder;
- }
- FieldType itemType = FieldUtil.determineFieldType(itemsNode);
- try {
- switch ( itemType ) {
- case OBJECT:
- builder = appendArrayObject(builder, schema, fieldId, itemsNode, seperator);
- break;
- case ARRAY:
- ObjectNode subArrayItems = (ObjectNode) itemsNode.get("items");
- builder = appendArrayItems(builder, schema, fieldId, subArrayItems, seperator);
- break;
- default:
- builder = appendArrayField(builder, schema, fieldId, itemType, seperator);
- }
- } catch (Exception ex) {
- LOGGER.warn("No item type resolvable for {}", fieldId);
- }
-
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendArrayField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- Objects.requireNonNull(fieldId);
- builder.append("{t: (");
- builder.append(pigEscape(fieldId));
- builder.append(seperator);
- builder.append(pigType(fieldType));
- builder.append(")}");
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendArrayObject(StringBuilder builder, Schema schema, String fieldId, ObjectNode fieldNode, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- Objects.requireNonNull(fieldId);
- Objects.requireNonNull(fieldNode);
- ObjectNode propertiesNode = schemaStore.resolveProperties(schema, fieldNode, fieldId);
- if ( propertiesNode.size() > 0 ) {
- builder.append("{t: (");
- builder = appendStructField(builder, schema, "", propertiesNode, ':');
- builder.append(")}");
- }
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private StringBuilder appendStructField(StringBuilder builder, Schema schema, String fieldId, ObjectNode propertiesNode, Character seperator) {
- // safe to append nothing
- Objects.requireNonNull(builder);
- Objects.requireNonNull(propertiesNode);
-
- if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) {
-
- currentDepth += 1;
-
- if (StringUtils.isNotBlank(fieldId)) {
- builder.append(pigEscape(fieldId));
- builder.append(seperator);
- builder.append("(");
- builder = appendPropertiesNode(builder, schema, propertiesNode, ':');
- builder.append(")");
- }
-
- currentDepth -= 1;
-
- }
- Objects.requireNonNull(builder);
- return builder;
- }
-
- private static String pigEscape( String fieldId ) {
- return fieldId;
- }
-
- private static String pigType( FieldType fieldType ) {
- switch ( fieldType ) {
- case STRING:
- return "chararray";
- case INTEGER:
- return "int";
- case NUMBER:
- return "double";
- case OBJECT:
- return "tuple";
- default:
- return fieldType.name().toLowerCase();
- }
- }
-
- private String schemaSymbol( Schema schema ) {
- if (schema == null) {
- return null;
- }
- // this needs to return whatever
- if (schema.getUri().getScheme().equals("file")) {
- String inputFile = schema.getUri().getPath();
- String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
- for (String sourcePath : config.getSourcePaths()) {
- resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
- }
- return dropExtension(resourcePath).replace("/", "_").replace("-", "");
- } else {
- return "IDK";
- }
- }
-}
diff --git a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java
deleted file mode 100644
index b1087c4..0000000
--- a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.pig;
-
-import org.apache.maven.plugin.AbstractMojo;
-import org.apache.maven.plugin.MojoExecutionException;
-import org.apache.maven.plugin.MojoFailureException;
-import org.apache.maven.plugins.annotations.Component;
-import org.apache.maven.plugins.annotations.Execute;
-import org.apache.maven.plugins.annotations.LifecyclePhase;
-import org.apache.maven.plugins.annotations.Mojo;
-import org.apache.maven.plugins.annotations.Parameter;
-import org.apache.maven.project.MavenProject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.List;
-
-@Mojo (
- name = "generate-resources",
- defaultPhase = LifecyclePhase.GENERATE_RESOURCES
- )
-@Execute (
- goal = "generate-resources",
- phase = LifecyclePhase.GENERATE_RESOURCES
- )
-public class StreamsPigResourceGeneratorMojo extends AbstractMojo {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsPigResourceGeneratorMojo.class);
-
- private volatile MojoFailureException mojoFailureException;
-
- @Component
- private MavenProject project;
-
- @Parameter( defaultValue = "${project.basedir}", readonly = true )
- private File basedir;
-
- @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only
- public String sourceDirectory;
-
- @Parameter( readonly = true ) // Maven 3 only
- public List<String> sourcePaths;
-
- @Parameter(defaultValue = "target/generated-resources/pig", readonly = true)
- public String targetDirectory;
-
- /**
- * execute StreamsPigResourceGeneratorMojo.
- * @throws MojoExecutionException MojoExecutionException
- * @throws MojoFailureException MojoFailureException
- */
- public void execute() throws MojoExecutionException, MojoFailureException {
-
- //addProjectDependenciesToClasspath();
-
- StreamsPigGenerationConfig config = new StreamsPigGenerationConfig();
-
- if ( sourcePaths != null && sourcePaths.size() > 0) {
- config.setSourcePaths(sourcePaths);
- } else {
- config.setSourceDirectory(sourceDirectory);
- }
- config.setTargetDirectory(targetDirectory);
-
- StreamsPigResourceGenerator streamsPigResourceGenerator = new StreamsPigResourceGenerator(config);
- streamsPigResourceGenerator.run();
- }
-
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pig/src/site/markdown/index.md b/streams-plugins/streams-plugin-pig/src/site/markdown/index.md
deleted file mode 100644
index 9cf39ef..0000000
--- a/streams-plugins/streams-plugin-pig/src/site/markdown/index.md
+++ /dev/null
@@ -1,38 +0,0 @@
-org.apache.streams.plugins:streams-plugin-pig
-=============================================
-
-streams-plugin-pig generates resources from json schemas to assist with analysis of json data using Apache Pig.
-
-### Usage
-
-Output will be placed in target/generated-resources/pig by default
-
-##### Maven
-
-Run within a module containing a src/main/jsonschema directory
-
- mvn org.apache.streams.plugins:streams-plugin-pig:0.4-incubating:pig
-
-[streams-plugin-pig/pom.xml](streams-plugin-pig/pom.xml "streams-plugin-pig/pom.xml")
-
-##### SDK
-
-Embed within your own java code
-
- StreamsPigGenerationConfig config = new StreamsPigGenerationConfig();
- config.setSourceDirectory("src/main/jsonschema");
- config.setTargetDirectory("target/generated-resources");
- StreamsPigResourceGenerator generator = new StreamsPigResourceGenerator(config);
- generator.run();
-
-##### CLI
-
-Run from CLI without Maven
-
- java -jar streams-plugin-pig-jar-with-dependencies.jar StreamsPigResourceGenerator src/main/jsonschema target/generated-resources
-
-#### Documentation
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pig/src/test/java/org/apache/streams/plugins/pig/test/StreamsPigResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-pig/src/test/java/org/apache/streams/plugins/pig/test/StreamsPigResourceGeneratorCLITest.java
deleted file mode 100644
index 9921ca0..0000000
--- a/streams-plugins/streams-plugin-pig/src/test/java/org/apache/streams/plugins/pig/test/StreamsPigResourceGeneratorCLITest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.pig.test;
-
-import org.apache.streams.plugins.pig.StreamsPigResourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Test;
-import org.testng.Assert;
-
-import java.io.File;
-import java.util.Collection;
-
-/**
- * Test whether StreamsPigResourceGeneratorCLI generates resources.
- */
-public class StreamsPigResourceGeneratorCLITest {
-
- @Test
- public void testStreamsPigResourceGeneratorCLI() throws Exception {
-
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
- String targetDirectory = "target/generated-resources/pig-cli";
-
- StreamsPigResourceGenerator.main(new String[]{sourceDirectory, targetDirectory});
-
- File testOutput = new File(targetDirectory);
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, StreamsPigResourceGeneratorTest.pigFilter, true);
- Assert.assertEquals(outputCollection.size(), 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pig/src/test/java/org/apache/streams/plugins/pig/test/StreamsPigResourceGeneratorMojoIT.java b/streams-plugins/streams-plugin-pig/src/test/java/org/apache/streams/plugins/pig/test/StreamsPigResourceGeneratorMojoIT.java
deleted file mode 100644
index f168c56..0000000
--- a/streams-plugins/streams-plugin-pig/src/test/java/org/apache/streams/plugins/pig/test/StreamsPigResourceGeneratorMojoIT.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.pig.test;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.maven.it.Verifier;
-import org.apache.maven.it.util.ResourceExtractor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests that streams-plugin-pig running via maven generates pig resources.
- */
-public class StreamsPigResourceGeneratorMojoIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsPigResourceGeneratorMojoIT.class);
-
- @Test
- public void testStreamsPigResourceGeneratorMojo() throws Exception {
-
- File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-pig" );
-
- Verifier verifier;
-
- verifier = new Verifier( testDir.getAbsolutePath() );
-
- List<String> cliOptions = new ArrayList<>();
- cliOptions.add( "-N" );
- verifier.executeGoals(Stream.of(
- "clean",
- "dependency:unpack-dependencies",
- "generate-resources").collect(Collectors.toList()));
-
- verifier.verifyErrorFreeLog();
-
- verifier.resetStreams();
-
- File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/pig-mojo");
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, StreamsPigResourceGeneratorTest.pigFilter, true);
- Assert.assertEquals(outputCollection.size(), 133 );
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pig/src/test/java/org/apache/streams/plugins/pig/test/StreamsPigResourceGeneratorTest.java b/streams-plugins/streams-plugin-pig/src/test/java/org/apache/streams/plugins/pig/test/StreamsPigResourceGeneratorTest.java
deleted file mode 100644
index 3c280bf..0000000
--- a/streams-plugins/streams-plugin-pig/src/test/java/org/apache/streams/plugins/pig/test/StreamsPigResourceGeneratorTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.pig.test;
-
-import org.apache.streams.plugins.pig.StreamsPigGenerationConfig;
-import org.apache.streams.plugins.pig.StreamsPigResourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-
-import java.io.File;
-import java.util.Collection;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Test that Activity beans are compatible with the example activities in the spec.
- */
-public class StreamsPigResourceGeneratorTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsPigResourceGeneratorTest.class);
-
- public static final String[] pigFilter = new String[]{"pig"};
-
- /**
- * Tests that StreamsPigResourceGenerator via SDK generates pig resources.
- *
- * @throws Exception Exception
- */
- @Test
- public void testStreamsPigResourceGenerator() throws Exception {
-
- StreamsPigGenerationConfig config = new StreamsPigGenerationConfig();
-
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
-
- config.setSourceDirectory(sourceDirectory);
-
- config.setTargetDirectory("target/generated-resources/pig");
-
- config.setExclusions(Stream.of("attachments").collect(Collectors.toSet()));
-
- config.setMaxDepth(2);
-
- StreamsPigResourceGenerator streamsPigResourceGenerator = new StreamsPigResourceGenerator(config);
- streamsPigResourceGenerator.run();
-
- File testOutput = config.getTargetDirectory();
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, pigFilter, true);
- Assert.assertEquals(outputCollection.size(), 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pig/src/test/resources/expected/media_link.pig b/streams-plugins/streams-plugin-pig/src/test/resources/expected/media_link.pig
deleted file mode 100644
index 94db813..0000000
--- a/streams-plugins/streams-plugin-pig/src/test/resources/expected/media_link.pig
+++ /dev/null
@@ -1,17 +0,0 @@
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
-media_link = LOAD '' USING JsonLoader('duration:double, height:int, width:int, url:chararray');
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pig/src/test/resources/expected/objectTypes/file.pig b/streams-plugins/streams-plugin-pig/src/test/resources/expected/objectTypes/file.pig
deleted file mode 100644
index b05a55f..0000000
--- a/streams-plugins/streams-plugin-pig/src/test/resources/expected/objectTypes/file.pig
+++ /dev/null
@@ -1,17 +0,0 @@
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
-objectTypes_file = LOAD '' USING JsonLoader('id:chararray, image:(duration:double, height:int, width:int, url:chararray), displayName:chararray, summary:chararray, content:chararray, url:chararray, objectType:chararray, author:(id:chararray, image:(duration:double, height:int, width:int, url:chararray), displayName:chararray, summary:chararray, content:chararray, url:chararray, objectType:chararray, author:(id:chararray, displayName:chararray, summary:chararray, content:chararray, url:ch [...]
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pig/src/test/resources/expected/objectTypes/photo-album.pig b/streams-plugins/streams-plugin-pig/src/test/resources/expected/objectTypes/photo-album.pig
deleted file mode 100644
index e4f075f..0000000
--- a/streams-plugins/streams-plugin-pig/src/test/resources/expected/objectTypes/photo-album.pig
+++ /dev/null
@@ -1,17 +0,0 @@
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
-objectTypes_photoalbum = LOAD '' USING JsonLoader('objectType:chararray, {t: ()}');
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pig/src/test/resources/streams-plugin-pig/pom.xml b/streams-plugins/streams-plugin-pig/src/test/resources/streams-plugin-pig/pom.xml
deleted file mode 100644
index 4fe6f49..0000000
--- a/streams-plugins/streams-plugin-pig/src/test/resources/streams-plugin-pig/pom.xml
+++ /dev/null
@@ -1,144 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pig-test</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Test StreamsPigResourceGeneratorMojo</name>
-
- <repositories>
- <repository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- </repositories>
-
- <pluginRepositories>
- <pluginRepository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </pluginRepository>
- <pluginRepository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </pluginRepository>
- </pluginRepositories>
-
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pig</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <configuration>
- <sourcePaths>
- <sourcePath>target/test-classes/activitystreams-schemas/activity.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/collection.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/media_link.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/object.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/objectTypes</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/verbs</sourcePath>
- </sourcePaths>
- <targetDirectory>target/generated-resources/pig-mojo</targetDirectory>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate-resources</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- </configuration>
- <executions>
- <execution>
- <id>unpack-schemas</id>
- <phase>process-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pojo/pom.xml b/streams-plugins/streams-plugin-pojo/pom.xml
deleted file mode 100644
index a9f580e..0000000
--- a/streams-plugins/streams-plugin-pojo/pom.xml
+++ /dev/null
@@ -1,205 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <packaging>maven-plugin</packaging>
-
- <parent>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugins</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-maven-plugin</artifactId>
- <version>${jsonschema2pojo.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.juneau</groupId>
- <artifactId>juneau-marshall</artifactId>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-aether-provider</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-artifact</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-compat</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.shared</groupId>
- <artifactId>maven-verifier</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-plugin-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.codehaus.plexus</groupId>
- <artifactId>plexus-utils</artifactId>
- <version>3.0.15</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-plugin-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-generators</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-testing</groupId>
- <artifactId>maven-plugin-testing-harness</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>org.joda</groupId>
- <artifactId>joda-convert</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <artifactId>maven-plugin-plugin</artifactId>
- <version>3.4</version>
- <configuration>
- <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
- </configuration>
- <executions>
- <execution>
- <id>mojo-descriptor</id>
- <phase>process-classes</phase>
- <goals>
- <goal>descriptor</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>activitystreams-schemas</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includes>**/*.json</includes>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- </configuration>
- </execution>
- <execution>
- <id>activitystreams-testdocs</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includes>**/*.json</includes>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>test-jar</includeTypes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-testdocs</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
-</project>
diff --git a/streams-plugins/streams-plugin-pojo/src/main/java/org/apache/streams/plugins/JuneauPojoAnnotator.java b/streams-plugins/streams-plugin-pojo/src/main/java/org/apache/streams/plugins/JuneauPojoAnnotator.java
deleted file mode 100644
index 437feb5..0000000
--- a/streams-plugins/streams-plugin-pojo/src/main/java/org/apache/streams/plugins/JuneauPojoAnnotator.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.sun.codemodel.JDefinedClass;
-import com.sun.codemodel.JFieldVar;
-import com.sun.codemodel.JMethod;
-import org.apache.juneau.annotation.BeanProperty;
-import org.jsonschema2pojo.AbstractAnnotator;
-
-/**
- * Add @BeanProperty annotations to getters, setters, anyGetter, and anySetter
- */
-public class JuneauPojoAnnotator extends AbstractAnnotator {
-
- /**
- * Add the necessary annotation to associate a Java field with a
- * JSON property
- *
- * @param field
- * the field that holds the value of the given JSON property
- * @param propertyName
- * the name of the JSON property that this field holds
- */
- public void propertyField(JFieldVar field, JDefinedClass clazz,
- String propertyName, JsonNode propertyNode) {
- field.annotate(BeanProperty.class).param("value", propertyName);
- }
-
-
- /**
- * Add the necessary annotation to mark a Java method as the getter for a
- * JSON property
- *
- * @param getter
- * the method that will be used to get the value of the given
- * JSON property
- * @param propertyName
- * the name of the JSON property that this getter gets
- */
- public void propertyGetter(JMethod getter, String propertyName) {
- getter.annotate(BeanProperty.class).param("value", propertyName);
- }
-
- /**
- * Add the necessary annotation to mark a Java method as the setter for a
- * JSON property
- *
- * @param setter
- * the method that will be used to set the value of the given
- * JSON property
- * @param propertyName
- * the name of the JSON property that this setter sets
- */
- public void propertySetter(JMethod setter, String propertyName) {
- setter.annotate(BeanProperty.class).param("value", propertyName);
- }
-
- /**
- * Add the necessary annotation to mark a Java method as the getter for
- * additional JSON property values that do not match any of the other
- * property names found in the bean.
- *
- * @param getter
- * the method that will be used to get the values of additional
- * properties
- */
- public void anyGetter(JMethod getter) {
- getter.annotate(BeanProperty.class).param("name", "*");
- }
-
- /**
- * Add the necessary annotation to mark a Java method as the setter for
- * additional JSON property values that do not match any of the other
- * property names found in the bean.
- *
- * @param setter
- * the method that will be used to set the values of additional
- * properties
- */
- public void anySetter(JMethod setter) {
- setter.annotate(BeanProperty.class).param("name", "*");
- }
-}
diff --git a/streams-plugins/streams-plugin-pojo/src/main/java/org/apache/streams/plugins/StreamsPojoGenerationConfig.java b/streams-plugins/streams-plugin-pojo/src/main/java/org/apache/streams/plugins/StreamsPojoGenerationConfig.java
deleted file mode 100644
index 4378b70..0000000
--- a/streams-plugins/streams-plugin-pojo/src/main/java/org/apache/streams/plugins/StreamsPojoGenerationConfig.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins;
-
-import org.jsonschema2pojo.Annotator;
-import org.jsonschema2pojo.DefaultGenerationConfig;
-import org.jsonschema2pojo.NoopAnnotator;
-import org.jsonschema2pojo.util.URLUtil;
-
-import java.io.File;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Configures StreamsPojoSourceGenerator.
- */
-public class StreamsPojoGenerationConfig extends DefaultGenerationConfig {
-
- private String sourceDirectory;
- private List<String> sourcePaths;
- private String targetPackage;
- private String targetDirectory;
-
- public void setSourceDirectory(String sourceDirectory) {
- this.sourceDirectory = sourceDirectory;
- }
-
- public void setSourcePaths(List<String> sourcePaths) {
- this.sourcePaths = sourcePaths;
- }
-
- public void setTargetPackage(String targetPackage) {
- this.targetPackage = targetPackage;
- }
-
- public void setTargetDirectory(String targetDirectory) {
- this.targetDirectory = targetDirectory;
- }
-
- @Override
- public String getTargetPackage() {
- return targetPackage;
- }
-
- @Override
- public File getTargetDirectory() {
- return new File(targetDirectory);
- }
-
- @Override
- public Iterator<URL> getSource() {
- if (null != sourceDirectory) {
- return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator();
- }
- List<URL> sourceUrls = new ArrayList<>();
- if ( sourcePaths != null && sourcePaths.size() > 0) {
- for (String source : sourcePaths) {
- sourceUrls.add(URLUtil.parseURL(source));
- }
- }
- return sourceUrls.iterator();
- }
-
- @Override
- public boolean isGenerateBuilders() {
- return true;
- }
-
- @Override
- public boolean isUseLongIntegers() {
- return true;
- }
-
- @Override
- public boolean isRemoveOldOutput() {
- return true;
- }
-
- @Override
- public boolean isUseJodaDates() {
- return true;
- }
-
- @Override
- public boolean isIncludeJsr303Annotations() {
- return true;
- }
-
- @Override
- public boolean isUseCommonsLang3() {
- return true;
- }
-
- // @Override
- // public boolean isIncludeAdditionalProperties() {
- // return true;
- // }
-
- @Override
- public Class<? extends Annotator> getCustomAnnotator() {
- return JuneauPojoAnnotator.class;
- }
-
-}
diff --git a/streams-plugins/streams-plugin-pojo/src/main/java/org/apache/streams/plugins/StreamsPojoSourceGenerator.java b/streams-plugins/streams-plugin-pojo/src/main/java/org/apache/streams/plugins/StreamsPojoSourceGenerator.java
deleted file mode 100644
index 0d31f10..0000000
--- a/streams-plugins/streams-plugin-pojo/src/main/java/org/apache/streams/plugins/StreamsPojoSourceGenerator.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins;
-
-import org.jsonschema2pojo.Jsonschema2Pojo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-import java.util.Objects;
-
-/**
- * Embed within your own java code
- *
- * <p/>
- * StreamsPojoGenerationConfig config = new StreamsPojoGenerationConfig();
- * config.setSourceDirectory("src/main/jsonschema");
- * config.setTargetDirectory("target/generated-sources/pojo");
- * StreamsPojoSourceGenerator generator = new StreamsPojoSourceGenerator(config);
- * generator.run();
- *
- */
-public class StreamsPojoSourceGenerator implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsPojoSourceGenerator.class);
-
- private static final String LS = System.getProperty("line.separator");
-
- private StreamsPojoGenerationConfig config;
-
- /**
- * Run from CLI without Maven
- *
- * <p/>
- * java -jar streams-plugin-pojo-jar-with-dependencies.jar StreamsPojoSourceGenerator src/main/jsonschema target/generated-sources
- *
- * @param args [sourceDirectory, targetDirectory, targetPackage]
- * */
- public static void main(String[] args) {
- StreamsPojoGenerationConfig config = new StreamsPojoGenerationConfig();
-
- String sourceDirectory = "src/main/jsonschema";
- String targetDirectory = "target/generated-sources/pojo";
- String targetPackage = "";
-
- if ( args.length > 0 ) {
- sourceDirectory = args[0];
- }
- if ( args.length > 1 ) {
- targetDirectory = args[1];
- }
- if ( args.length > 2 ) {
- targetPackage = args[2];
- }
-
- config.setSourceDirectory(sourceDirectory);
- config.setTargetPackage(targetPackage);
- config.setTargetDirectory(targetDirectory);
-
- StreamsPojoSourceGenerator streamsPojoSourceGenerator = new StreamsPojoSourceGenerator(config);
- streamsPojoSourceGenerator.run();
- }
-
- public StreamsPojoSourceGenerator(StreamsPojoGenerationConfig config) {
- this.config = config;
- }
-
- @Override
- public void run() {
-
- Objects.requireNonNull(config);
-
- try {
- Jsonschema2Pojo.generate(config);
- } catch (Throwable ex) {
- LOGGER.error("{} {}", ex.getClass(), ex.getMessage());
- }
- }
-
- private void writeFile(String pojoFile, String pojoHive) {
- try {
- File path = new File(pojoFile);
- File dir = path.getParentFile();
- if ( !dir.exists() ) {
- dir.mkdirs();
- }
- Files.write(Paths.get(pojoFile), pojoHive.getBytes(), StandardOpenOption.CREATE_NEW);
- } catch (Exception ex) {
- LOGGER.error("Write Exception: {}", ex);
- }
- }
-}
diff --git a/streams-plugins/streams-plugin-pojo/src/main/java/org/apache/streams/plugins/StreamsPojoSourceGeneratorMojo.java b/streams-plugins/streams-plugin-pojo/src/main/java/org/apache/streams/plugins/StreamsPojoSourceGeneratorMojo.java
deleted file mode 100644
index 7585e7b..0000000
--- a/streams-plugins/streams-plugin-pojo/src/main/java/org/apache/streams/plugins/StreamsPojoSourceGeneratorMojo.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins;
-
-import org.apache.maven.artifact.DependencyResolutionRequiredException;
-import org.apache.maven.plugin.AbstractMojo;
-import org.apache.maven.plugin.MojoExecutionException;
-import org.apache.maven.plugin.MojoFailureException;
-import org.apache.maven.plugins.annotations.Component;
-import org.apache.maven.plugins.annotations.Execute;
-import org.apache.maven.plugins.annotations.LifecyclePhase;
-import org.apache.maven.plugins.annotations.Mojo;
-import org.apache.maven.plugins.annotations.Parameter;
-import org.apache.maven.project.MavenProject;
-import org.jsonschema2pojo.maven.ProjectClasspath;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.List;
-
-@Mojo (
- name = "generate-sources",
- defaultPhase = LifecyclePhase.GENERATE_SOURCES
- )
-@Execute (
- goal = "generate-sources",
- phase = LifecyclePhase.GENERATE_SOURCES
- )
-public class StreamsPojoSourceGeneratorMojo extends AbstractMojo {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsPojoSourceGeneratorMojo.class);
-
- private volatile MojoFailureException mojoFailureException;
-
- @Component
- public MavenProject project;
-
- @Parameter( defaultValue = "${project.basedir}", readonly = true )
- public File basedir;
-
- @Parameter( defaultValue = "./src/main/jsonschema", readonly = true ) // Maven 3 only
- public String sourceDirectory;
-
- @Parameter( readonly = true ) // Maven 3 only
- public List<String> sourcePaths;
-
- @Parameter(defaultValue = "./target/generated-sources/pojo", readonly = true)
- public String targetDirectory;
-
- @Parameter(readonly = true)
- public String targetPackage;
-
- /**
- * execute StreamsPojoSourceGenerator.
- * @throws MojoExecutionException MojoExecutionException
- * @throws MojoFailureException MojoFailureException
- */
- public void execute() throws MojoExecutionException, MojoFailureException {
-
- addProjectDependenciesToClasspath();
-
- StreamsPojoGenerationConfig config = new StreamsPojoGenerationConfig();
-
- if ( sourcePaths != null && sourcePaths.size() > 0) {
- config.setSourcePaths(sourcePaths);
- } else {
- config.setSourceDirectory(sourceDirectory);
- }
- config.setTargetPackage(targetPackage);
- config.setTargetDirectory(targetDirectory);
-
- StreamsPojoSourceGenerator streamsPojoSourceGenerator = new StreamsPojoSourceGenerator(config);
- streamsPojoSourceGenerator.run();
-
- }
-
- private void addProjectDependenciesToClasspath() {
-
- try {
-
- ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader();
- ClassLoader newClassLoader = new ProjectClasspath().getClassLoader(project, oldClassLoader, getLog());
- Thread.currentThread().setContextClassLoader(newClassLoader);
-
- } catch (DependencyResolutionRequiredException ex) {
- LOGGER.info("Skipping addition of project artifacts, there appears to be a dependecy resolution problem", ex);
- }
-
- }
-
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pojo/src/site/markdown/index.md b/streams-plugins/streams-plugin-pojo/src/site/markdown/index.md
deleted file mode 100644
index 514a693..0000000
--- a/streams-plugins/streams-plugin-pojo/src/site/markdown/index.md
+++ /dev/null
@@ -1,38 +0,0 @@
-org.apache.streams.plugins:streams-plugin-pojo
-==============================================
-
-streams-plugin-pojo generates source files from json schemas suitable for writing Apache Streams components and libraries in Java.
-
-### Usage
-
-Output will be placed in target/generated-sources/pojo by default
-
-##### Maven
-
-Run within a module containing a src/main/jsonschema directory
-
- mvn org.apache.streams.plugins:streams-plugin-pojo:0.4-incubating:pojo
-
-[streams-plugin-pojo/pom.xml](streams-plugin-pojo/pom.xml "streams-plugin-pojo/pom.xml")
-
-##### SDK
-
-Embed within your own java code
-
- StreamsPojoGenerationConfig config = new StreamsPojoGenerationConfig();
- config.setSourceDirectory("src/main/jsonschema");
- config.setTargetDirectory("target/generated-resources");
- StreamsPojoSourceGenerator generator = new StreamsPojoSourceGenerator(config);
- generator.run();
-
-##### CLI
-
-Run from CLI without Maven
-
- java -jar streams-plugin-pojo-jar-with-dependencies.jar StreamsPojoSourceGenerator src/main/jsonschema target/generated-sources
-
-#### Documentation
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pojo/src/test/java/org/apache/streams/plugins/test/StreamsPojoSourceGeneratorCLITest.java b/streams-plugins/streams-plugin-pojo/src/test/java/org/apache/streams/plugins/test/StreamsPojoSourceGeneratorCLITest.java
deleted file mode 100644
index 1c72626..0000000
--- a/streams-plugins/streams-plugin-pojo/src/test/java/org/apache/streams/plugins/test/StreamsPojoSourceGeneratorCLITest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.test;
-
-import org.apache.streams.plugins.StreamsPojoSourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.Collection;
-
-/**
- * Test whether StreamsPojoSourceGeneratorCLI generates source files.
- */
-public class StreamsPojoSourceGeneratorCLITest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsPojoSourceGeneratorCLITest.class);
-
- @Test
- public void testStreamsPojoSourceGeneratorCLI() throws Exception {
-
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
- String targetDirectory = "target/generated-sources/test-cli";
-
- StreamsPojoSourceGenerator.main(new String[]{sourceDirectory, targetDirectory});
-
- File testOutput = new File(targetDirectory);
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> testOutputFiles = FileUtils.listFiles(testOutput, StreamsPojoSourceGeneratorTest.javaFilter, true);
- Assert.assertTrue(testOutputFiles.size() > 133);
- }
-}
diff --git a/streams-plugins/streams-plugin-pojo/src/test/java/org/apache/streams/plugins/test/StreamsPojoSourceGeneratorMojoIT.java b/streams-plugins/streams-plugin-pojo/src/test/java/org/apache/streams/plugins/test/StreamsPojoSourceGeneratorMojoIT.java
deleted file mode 100644
index 96cd066..0000000
--- a/streams-plugins/streams-plugin-pojo/src/test/java/org/apache/streams/plugins/test/StreamsPojoSourceGeneratorMojoIT.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.test;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.maven.it.Verifier;
-import org.apache.maven.it.util.ResourceExtractor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests that streams-plugin-pojo running via maven can convert activity schemas into pojos
- * which then compile.
- */
-@Test
-public class StreamsPojoSourceGeneratorMojoIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsPojoSourceGeneratorMojoIT.class);
-
-// @BeforeClass
-// protected void setUp() throws Exception {
-// // required for mojo lookups to work
-// super.setUp();
-// }
-
-
- @Test
- public void testStreamsPojoSourceGeneratorMojo() throws Exception {
-
- File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-pojo" );
-
- Verifier verifier;
-
- verifier = new Verifier( testDir.getAbsolutePath() );
-
- List<String> cliOptions = new ArrayList<>();
- cliOptions.add( "-N" );
- verifier.executeGoals(Stream.of(
- "clean",
- "dependency:unpack-dependencies",
- "generate-sources",
- "compile").collect(Collectors.toList()));
-
- verifier.verifyErrorFreeLog();
-
- verifier.resetStreams();
-
- File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-sources/pojo-mojo");
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> testOutputFiles = FileUtils.listFiles(testOutput, StreamsPojoSourceGeneratorTest.javaFilter, true);
- Assert.assertTrue( testOutputFiles.size() > 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pojo/src/test/java/org/apache/streams/plugins/test/StreamsPojoSourceGeneratorTest.java b/streams-plugins/streams-plugin-pojo/src/test/java/org/apache/streams/plugins/test/StreamsPojoSourceGeneratorTest.java
deleted file mode 100644
index bb4b608..0000000
--- a/streams-plugins/streams-plugin-pojo/src/test/java/org/apache/streams/plugins/test/StreamsPojoSourceGeneratorTest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.test;
-
-import org.apache.streams.plugins.StreamsPojoGenerationConfig;
-import org.apache.streams.plugins.StreamsPojoSourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.Collection;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests that StreamsPojoSourceGenerator via SDK generates java sources.
- *
- */
-public class StreamsPojoSourceGeneratorTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsPojoSourceGeneratorTest.class);
-
- public static final String[] javaFilter = new String[]{"java"};
-
- /**
- * Tests that StreamsPojoSourceGenerator via SDK generates pig resources.
- *
- * @throws Exception Exception
- */
- @Test
- public void testStreamsPojoSourceGenerator() throws Exception {
-
- StreamsPojoGenerationConfig config = new StreamsPojoGenerationConfig();
-
- List<String> sourcePaths = Stream.of(
- "target/test-classes/activitystreams-schemas/activity.json",
- "target/test-classes/activitystreams-schemas/collection.json",
- "target/test-classes/activitystreams-schemas/media_link.json",
- "target/test-classes/activitystreams-schemas/object.json",
- "target/test-classes/activitystreams-schemas/objectTypes",
- "target/test-classes/activitystreams-schemas/verbs"
- ).collect(Collectors.toList());
- config.setSourcePaths(sourcePaths);
-
- config.setTargetPackage("org.apache.streams.pojo");
- config.setTargetDirectory("target/generated-sources/pojo");
-
- StreamsPojoSourceGenerator streamsPojoSourceGenerator = new StreamsPojoSourceGenerator(config);
- streamsPojoSourceGenerator.run();
-
- Assert.assertNotNull(config.getTargetDirectory());
- Assert.assertTrue(config.getTargetDirectory().exists());
- Assert.assertTrue(config.getTargetDirectory().isDirectory());
-
- Collection<File> targetFiles = FileUtils.listFiles(config.getTargetDirectory(), javaFilter, true);
- Assert.assertTrue(targetFiles.size() > 133);
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-pojo/src/test/resources/streams-plugin-pojo/pom.xml b/streams-plugins/streams-plugin-pojo/src/test/resources/streams-plugin-pojo/pom.xml
deleted file mode 100644
index 7b01242..0000000
--- a/streams-plugins/streams-plugin-pojo/src/test/resources/streams-plugin-pojo/pom.xml
+++ /dev/null
@@ -1,199 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo-test</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Test StreamsPojoMojo</name>
-
- <properties>
- <jackson.version>2.6.1</jackson.version>
- <commons-lang3.version>3.4</commons-lang3.version>
- </properties>
-
- <repositories>
- <repository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- </repositories>
-
- <pluginRepositories>
- <pluginRepository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </pluginRepository>
- <pluginRepository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </pluginRepository>
- </pluginRepositories>
-
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>jar</type>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-joda</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>${commons-lang3.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>target/test-classes/activitystreams-schemas/activity.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/collection.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/media_link.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/object.json</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/objectTypes</sourcePath>
- <sourcePath>target/test-classes/activitystreams-schemas/verbs</sourcePath>
- </sourcePaths>
- <targetPackage>org.apache.streams.pojo.mojo</targetPackage>
- <targetDirectory>target/generated-sources/pojo-mojo</targetDirectory>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate-sources</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- </configuration>
- <executions>
- <execution>
- <id>unpack-schemas</id>
- <phase>process-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.8</version>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo-mojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-scala/pom.xml b/streams-plugins/streams-plugin-scala/pom.xml
deleted file mode 100644
index 32375b1..0000000
--- a/streams-plugins/streams-plugin-scala/pom.xml
+++ /dev/null
@@ -1,224 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
- <artifactId>streams-plugin-scala</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <packaging>maven-plugin</packaging>
-
- <parent>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugins</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-aether-provider</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-artifact</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-compat</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.shared</groupId>
- <artifactId>maven-verifier</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-plugin-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.codehaus.plexus</groupId>
- <artifactId>plexus-utils</artifactId>
- <version>3.0.15</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-plugin-plugin</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-tools-generators</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-testing</groupId>
- <artifactId>maven-plugin-testing-harness</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>org.joda</groupId>
- <artifactId>joda-convert</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <artifactId>maven-plugin-plugin</artifactId>
- <version>3.4</version>
- <configuration>
- <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
- </configuration>
- <executions>
- <execution>
- <id>mojo-descriptor</id>
- <phase>process-classes</phase>
- <goals>
- <goal>descriptor</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>streams-pojo-resource-dependencies</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-pojo</includeArtifactIds>
- <includes>org/apache/streams/pojo/json/**</includes>
- <outputDirectory>${project.build.directory}/test-classes/streams-pojo</outputDirectory>
- </configuration>
- </execution>
- <execution>
- <id>activitystreams-testdocs</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includes>**/*.json</includes>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>test-jar</includeTypes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-testdocs</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>scala-compile-generated-sources</id>
- <phase>integration-test</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- <configuration>
- <sourceDir>target/generated-sources/scala</sourceDir>
- <target>target/testClasses/scala</target>
- </configuration>
- </execution>
- </executions>
- <configuration>
- <scalaVersion>${scala.version}</scalaVersion>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
-</project>
diff --git a/streams-plugins/streams-plugin-scala/src/main/java/org/apache/streams/plugins/StreamsScalaGenerationConfig.java b/streams-plugins/streams-plugin-scala/src/main/java/org/apache/streams/plugins/StreamsScalaGenerationConfig.java
deleted file mode 100644
index f6ee814..0000000
--- a/streams-plugins/streams-plugin-scala/src/main/java/org/apache/streams/plugins/StreamsScalaGenerationConfig.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins;
-
-import java.io.File;
-import java.util.List;
-
-/**
- * Configures StreamsScalaSourceGenerator.
- */
-public class StreamsScalaGenerationConfig {
-
- private List<String> sourcePackages;
- private String targetPackage;
- private String targetDirectory;
-
- public void setSourcePackages(List<String> sourcePackages) {
- this.sourcePackages = sourcePackages;
- }
-
- public List<String> getSourcePackages() {
- return sourcePackages;
- }
-
- public void setTargetPackage(String targetPackage) {
- this.targetPackage = targetPackage;
- }
-
- public void setTargetDirectory(String targetDirectory) {
- this.targetDirectory = targetDirectory;
- }
-
- public String getTargetPackage() {
- return targetPackage;
- }
-
- public File getTargetDirectory() {
- return new File(targetDirectory);
- }
-
-}
diff --git a/streams-plugins/streams-plugin-scala/src/main/java/org/apache/streams/plugins/StreamsScalaSourceGenerator.java b/streams-plugins/streams-plugin-scala/src/main/java/org/apache/streams/plugins/StreamsScalaSourceGenerator.java
deleted file mode 100644
index b820125..0000000
--- a/streams-plugins/streams-plugin-scala/src/main/java/org/apache/streams/plugins/StreamsScalaSourceGenerator.java
+++ /dev/null
@@ -1,422 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins;
-
-import org.reflections.ReflectionUtils;
-import org.reflections.Reflections;
-import org.reflections.scanners.SubTypesScanner;
-import org.reflections.scanners.TypeAnnotationsScanner;
-import org.reflections.util.ConfigurationBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.Serializable;
-import java.lang.reflect.Field;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Embed within your own java code
- *
- * <p/>
- * StreamsScalaGenerationConfig config = new StreamsScalaGenerationConfig();
- * config.setTargetDirectory("target/generated-sources/scala");
- * config.setTargetPackage("com.example");
- * StreamsScalaSourceGenerator generator = new StreamsScalaSourceGenerator(config);
- * generator.run();
- *
- */
-public class StreamsScalaSourceGenerator implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsScalaSourceGenerator.class);
-
- private static final String LS = System.getProperty("line.separator");
-
- private StreamsScalaGenerationConfig config;
-
- private Reflections reflections;
-
- private String outDir;
-
- /**
- * Run from CLI without Maven
- *
- * <p/>
- * java -jar streams-plugin-scala-jar-with-dependencies.jar StreamsScalaSourceGenerator target/generated-sources
- *
- * @param args [targetDirectory, targetPackage]
- * */
- public static void main(String[] args) {
- StreamsScalaGenerationConfig config = new StreamsScalaGenerationConfig();
-
- List<String> sourcePackages = new ArrayList<>();
- String targetDirectory = "target/generated-sources/pojo";
- String targetPackage = "";
-
- if ( args.length > 0 ) {
- sourcePackages = Stream.of(args[0].split(",")).collect(Collectors.toList());
- }
- if ( args.length > 1 ) {
- targetDirectory = args[1];
- }
- if ( args.length > 2 ) {
- targetPackage = args[2];
- }
-
- config.setSourcePackages(sourcePackages);
- config.setTargetPackage(targetPackage);
- config.setTargetDirectory(targetDirectory);
-
- StreamsScalaSourceGenerator streamsScalaSourceGenerator = new StreamsScalaSourceGenerator(config);
- streamsScalaSourceGenerator.run();
- }
-
- /**
- * StreamsScalaSourceGenerator constructor.
- * @param config StreamsScalaGenerationConfig
- */
- public StreamsScalaSourceGenerator(StreamsScalaGenerationConfig config) {
- this.config = config;
- this.outDir = config.getTargetDirectory().getAbsolutePath();
- reflections = new Reflections(
- new ConfigurationBuilder()
- // TODO
- .forPackages(
- config.getSourcePackages()
- .toArray(new String[config.getSourcePackages().size()])
- )
- .setScanners(
- new SubTypesScanner(),
- new TypeAnnotationsScanner()));
-
- }
-
- @Override
- public void run() {
-
- List<Class<?>> serializableClasses = detectSerializableClasses();
-
- LOGGER.info("Detected {} serialiables:", serializableClasses.size());
- for ( Class clazz : serializableClasses ) {
- LOGGER.debug(clazz.toString());
- }
-
- List<Class<?>> pojoClasses = detectPojoClasses(serializableClasses);
-
- LOGGER.info("Detected {} pojos:", pojoClasses.size());
- for ( Class clazz : pojoClasses ) {
- LOGGER.debug(clazz.toString());
- }
-
- List<Class<?>> traits = detectTraits(pojoClasses);
-
- LOGGER.info("Detected {} traits:", traits.size());
- for ( Class clazz : traits ) {
- LOGGER.debug(clazz.toString());
- }
-
- List<Class<?>> cases = detectCases(pojoClasses);
-
- LOGGER.info("Detected {} cases:", cases.size());
- for ( Class clazz : cases ) {
- LOGGER.debug(clazz.toString());
- }
-
- for ( Class clazz : traits ) {
- String pojoPath = clazz.getPackage().getName().replace(".pojo.json", ".scala").replace(".","/") + "/traits/";
- String pojoName = clazz.getSimpleName() + ".scala";
- String pojoScala = renderTrait(clazz);
- writeFile(outDir + "/" + pojoPath + pojoName, pojoScala);
- }
-
- for ( Class clazz : traits ) {
- String pojoPath = clazz.getPackage().getName().replace(".pojo.json", ".scala").replace(".","/") + "/";
- String pojoName = clazz.getSimpleName() + ".scala";
- String pojoScala = renderClass(clazz);
- writeFile(outDir + "/" + pojoPath + pojoName, pojoScala);
- }
-
- for ( Class clazz : cases ) {
- String pojoPath = clazz.getPackage().getName().replace(".pojo.json", ".scala").replace(".","/") + "/";
- String pojoName = clazz.getSimpleName() + ".scala";
- String pojoScala = renderCase(clazz);
- writeFile(outDir + "/" + pojoPath + pojoName, pojoScala);
- }
-
- }
-
- private void writeFile(String pojoFile, String pojoScala) {
- try {
- File path = new File(pojoFile);
- File dir = path.getParentFile();
- if ( !dir.exists() ) {
- dir.mkdirs();
- }
- Files.write(Paths.get(pojoFile), pojoScala.getBytes(), StandardOpenOption.CREATE_NEW);
- } catch (Exception ex) {
- LOGGER.error("Write Exception: {}", ex);
- }
- }
-
- /**
- * detectSerializableClasses.
- * @return List of Serializable Classes
- */
- public List<Class<?>> detectSerializableClasses() {
-
- Set<Class<? extends Serializable>> classes =
- reflections.getSubTypesOf(java.io.Serializable.class);
-
- List<Class<?>> result = new ArrayList<>();
-
- for ( Class clazz : classes ) {
- result.add(clazz);
- }
-
- return result;
- }
-
- /**
- * detect which Classes are Pojo Classes.
- * @param classes List of candidate Pojo Classes
- * @return List of actual Pojo Classes
- */
- public List<Class<?>> detectPojoClasses(List<Class<?>> classes) {
-
- List<Class<?>> result = new ArrayList<>();
-
- for ( Class clazz : classes ) {
- try {
- clazz.newInstance().toString();
- } catch ( Exception ex) {
- //
- }
- // super-halfass way to know if this is a jsonschema2pojo
- if ( clazz.getAnnotations().length >= 1 ) {
- result.add(clazz);
- }
- }
-
- return result;
- }
-
- private List<Class<?>> detectTraits(List<Class<?>> classes) {
-
- List<Class<?>> traits = new ArrayList<>();
-
- for ( Class clazz : classes ) {
- if (reflections.getSubTypesOf(clazz).size() > 0) {
- traits.add(clazz);
- }
- }
-
- return traits;
- }
-
- private List<Class<?>> detectCases(List<Class<?>> classes) {
-
- List<Class<?>> cases = new ArrayList<>();
-
- for ( Class clazz : classes ) {
- if (reflections.getSubTypesOf(clazz).size() == 0) {
- cases.add(clazz);
- }
- }
-
- return cases;
- }
-
- private String renderTrait(Class<?> pojoClass) {
- StringBuffer stringBuffer = new StringBuffer();
- stringBuffer.append("package ");
- stringBuffer.append(pojoClass.getPackage().getName().replace(".pojo.json", ".scala"));
- stringBuffer.append(".traits");
- stringBuffer.append(LS);
- stringBuffer.append("trait ").append(pojoClass.getSimpleName());
- stringBuffer.append(" extends Serializable");
- stringBuffer.append(" {");
-
- Set<Field> fields = ReflectionUtils.getAllFields(pojoClass);
- appendFields(stringBuffer, fields, "def", ";");
-
- stringBuffer.append("}");
-
- return stringBuffer.toString();
- }
-
- private String renderClass(Class<?> pojoClass) {
- StringBuffer stringBuffer = new StringBuffer();
- stringBuffer.append("package ");
- stringBuffer.append(pojoClass.getPackage().getName().replace(".pojo.json", ".scala"));
- stringBuffer.append(LS);
- stringBuffer.append("import org.apache.commons.lang.builder.{HashCodeBuilder, EqualsBuilder, ToStringBuilder}");
- stringBuffer.append(LS);
- stringBuffer.append("class ").append(pojoClass.getSimpleName());
- stringBuffer.append(" (");
-
- Set<Field> fields = ReflectionUtils.getAllFields(pojoClass);
- appendFields(stringBuffer, fields, "var", ",");
-
- stringBuffer.append(")");
- stringBuffer.append(" extends ").append(pojoClass.getPackage().getName().replace(".pojo.json", ".scala")).append(".traits.").append(pojoClass.getSimpleName());
- stringBuffer.append(" with Serializable ");
- stringBuffer.append("{ ");
- stringBuffer.append(LS);
- stringBuffer.append("override def equals(obj: Any) = obj match { ");
- stringBuffer.append(LS);
- stringBuffer.append(" case other: ");
- stringBuffer.append(pojoClass.getSimpleName());
- stringBuffer.append(" => other.getClass == getClass && EqualsBuilder.reflectionEquals(this,obj)");
- stringBuffer.append(LS);
- stringBuffer.append(" case _ => false");
- stringBuffer.append(LS);
- stringBuffer.append("}");
- stringBuffer.append(LS);
- stringBuffer.append("override def hashCode = new HashCodeBuilder().hashCode");
- stringBuffer.append(LS);
- stringBuffer.append("}");
-
- return stringBuffer.toString();
- }
-
- private String renderCase(Class<?> pojoClass) {
- StringBuffer stringBuffer = new StringBuffer();
- stringBuffer.append("package ");
- stringBuffer.append(pojoClass.getPackage().getName().replace(".pojo.json", ".scala"));
- stringBuffer.append(LS);
- stringBuffer.append("case class " + pojoClass.getSimpleName());
- stringBuffer.append("(");
- Set<Field> fields = ReflectionUtils.getAllFields(pojoClass);
- appendFields(stringBuffer, fields, "var", ",");
- stringBuffer.append(")");
- if ( pojoClass.getSuperclass() != null && !pojoClass.getSuperclass().equals(java.lang.Object.class)) {
- stringBuffer.append(" extends " + pojoClass.getSuperclass().getPackage().getName().replace(".pojo.json", ".scala") + ".traits." + pojoClass.getSuperclass().getSimpleName());
- }
- stringBuffer.append(LS);
-
- return stringBuffer.toString();
- }
-
- private void appendFields(StringBuffer stringBuffer, Set<Field> fields, String varDef, String fieldDelimiter) {
- if ( fields.size() > 0 ) {
- stringBuffer.append(LS);
- Map<String,Field> fieldsToAppend = uniqueFields(fields);
- for ( Iterator<Field> iter = fieldsToAppend.values().iterator(); iter.hasNext(); ) {
- Field field = iter.next();
- if ( override( field ) ) {
- stringBuffer.append("override ");
- }
- stringBuffer.append(varDef);
- stringBuffer.append(" ");
- stringBuffer.append(name(field));
- stringBuffer.append(": ");
- if ( option(field) ) {
- stringBuffer.append("scala.Option[");
- stringBuffer.append(type(field));
- stringBuffer.append("]");
- } else {
- stringBuffer.append(type(field));
- }
- if ( !fieldDelimiter.equals(";") && value(field) != null) {
- stringBuffer.append(" = ");
- if ( option(field) ) {
- stringBuffer.append("scala.Some(");
- stringBuffer.append(value(field));
- stringBuffer.append(")");
- } else {
- stringBuffer.append(value(field));
- }
- }
- if ( iter.hasNext()) {
- stringBuffer.append(fieldDelimiter);
- }
- stringBuffer.append(LS);
- }
- } else {
- stringBuffer.append(LS);
- }
- }
-
- private boolean option(Field field) {
- return !field.getName().equals("verb") &&
- !field.getType().equals(Map.class) &&
- !field.getType().equals(List.class);
- }
-
- private String value(Field field) {
- switch (field.getName()) {
- case "verb":
- return "\"post\"";
- case "objectType":
- return "\"application\"";
- default:
- return null;
- }
- }
-
- private String type(Field field) {
- if ( field.getType().equals(java.lang.String.class)) {
- return "String";
- } else if ( field.getType().equals(java.util.Map.class)) {
- return "scala.collection.mutable.Map[String,Any]";
- } else if ( field.getType().equals(java.util.List.class)) {
- return "scala.collection.mutable.MutableList[Any]";
- }
- return field.getType().getCanonicalName().replace(".pojo.json", ".scala");
- }
-
- private Map<String,Field> uniqueFields(Set<Field> fieldset) {
- Map<String,Field> fields = new TreeMap<>();
- Field item = null;
- for ( Iterator<Field> it = fieldset.iterator(); it.hasNext(); item = it.next() ) {
- if ( item != null && item.getName() != null ) {
- Field added = fields.put(item.getName(), item);
- }
- // ensure right class will get used
- }
- return fields;
- }
-
- private String name(Field field) {
- if ( field.getName().equals("object")) {
- return "obj";
- } else {
- return field.getName();
- }
- }
-
- private boolean override(Field field) {
- try {
- return field.getDeclaringClass().getSuperclass().getField(field.getName()) != null;
- } catch ( Exception ex ) {
- return false;
- }
- }
-}
diff --git a/streams-plugins/streams-plugin-scala/src/main/java/org/apache/streams/plugins/StreamsScalaSourceGeneratorMojo.java b/streams-plugins/streams-plugin-scala/src/main/java/org/apache/streams/plugins/StreamsScalaSourceGeneratorMojo.java
deleted file mode 100644
index 44c82ac..0000000
--- a/streams-plugins/streams-plugin-scala/src/main/java/org/apache/streams/plugins/StreamsScalaSourceGeneratorMojo.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins;
-
-import com.google.common.base.Splitter;
-import org.apache.maven.plugin.AbstractMojo;
-import org.apache.maven.plugin.MojoExecutionException;
-import org.apache.maven.plugin.MojoFailureException;
-import org.apache.maven.plugins.annotations.Component;
-import org.apache.maven.plugins.annotations.Execute;
-import org.apache.maven.plugins.annotations.LifecyclePhase;
-import org.apache.maven.plugins.annotations.Mojo;
-import org.apache.maven.plugins.annotations.Parameter;
-import org.apache.maven.project.MavenProject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-
-@Mojo(
- name = "scala",
- defaultPhase = LifecyclePhase.GENERATE_SOURCES
- )
-@Execute(
- goal = "scala",
- phase = LifecyclePhase.GENERATE_SOURCES
- )
-public class StreamsScalaSourceGeneratorMojo extends AbstractMojo {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsScalaSourceGeneratorMojo.class);
-
- @Component
- private MavenProject project;
-
- // @Component
- // private Settings settings;
- //
- // @Parameter( defaultValue = "${localRepository}", readonly = true, required = true )
- // protected ArtifactRepository localRepository;
- //
- // @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only
- // private PluginDescriptor plugin;
- //
- @Parameter( defaultValue = "${project.basedir}", readonly = true )
- private File basedir;
-
- @Parameter(defaultValue = "${project.build.directory}", readonly = true)
- private File target;
-
- @Parameter(defaultValue = "org.apache.streams.pojo.json", readonly = true)
- private String packages;
-
- /**
- * execute StreamsScalaSourceGeneratorMojo.
- * @throws MojoExecutionException MojoExecutionException
- * @throws MojoFailureException MojoFailureException
- */
- public void execute() throws MojoExecutionException {
- StreamsScalaGenerationConfig config = new StreamsScalaGenerationConfig();
- config.setSourcePackages(Splitter.on(',').splitToList(packages));
- config.setTargetDirectory(target.toString());
-
- StreamsScalaSourceGenerator streamsScalaSourceGenerator = new StreamsScalaSourceGenerator(config);
-
- streamsScalaSourceGenerator.run();
- }
-
- public File getTarget() {
- return target;
- }
-
- public String getPackages() {
- return packages;
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-scala/src/site/markdown/index.md b/streams-plugins/streams-plugin-scala/src/site/markdown/index.md
deleted file mode 100644
index 76b142f..0000000
--- a/streams-plugins/streams-plugin-scala/src/site/markdown/index.md
+++ /dev/null
@@ -1,36 +0,0 @@
-org.apache.streams.plugins:streams-plugin-scala
-=============================================
-
-streams-plugin-scala generates source files from json schemas suitable for writing Apache Streams components and libraries in Scala.
-
-### Usage
-
-##### Maven
-
-Run within a module containing a src/main/jsonschema directory
-
- mvn org.apache.streams.plugins:streams-plugin-scala:0.4-incubating:pig
-
-[streams-plugin-scala/pom.xml](streams-plugin-scala/pom.xml "streams-plugin-scala/pom.xml")
-
-##### SDK
-
-Embed within your own java code
-
- StreamsScalaGenerationConfig config = new StreamsScalaGenerationConfig();
- config.setSourceDirectory("src/main/jsonschema");
- config.setTargetDirectory("target/generated-resources");
- StreamsScalaSourceGenerator generator = new StreamsScalaSourceGenerator(config);
- generator.run();
-
-##### CLI
-
-Run from CLI without Maven
-
- java -jar streams-plugin-scala-jar-with-dependencies.jar StreamsScalaSourceGenerator src/main/jsonschema target/generated-sources
-
-#### Documentation
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-scala/src/test/java/org/apache/streams/plugins/test/StreamsScalaSourceGeneratorCLITest.java b/streams-plugins/streams-plugin-scala/src/test/java/org/apache/streams/plugins/test/StreamsScalaSourceGeneratorCLITest.java
deleted file mode 100644
index f1e344e..0000000
--- a/streams-plugins/streams-plugin-scala/src/test/java/org/apache/streams/plugins/test/StreamsScalaSourceGeneratorCLITest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.test;
-
-import org.apache.streams.plugins.StreamsScalaSourceGenerator;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.Collection;
-
-import static org.apache.streams.plugins.test.StreamsScalaSourceGeneratorTest.scalaFilter;
-
-/**
- * Test whether StreamsScalaSourceGeneratorCLI generates sources.
- */
-public class StreamsScalaSourceGeneratorCLITest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsScalaSourceGeneratorCLITest.class);
-
- @Test
- public void testStreamsScalaSourceGeneratorCLI() throws Exception {
-
- String sourcePackages = "org.apache.streams.pojo.json";
- String targetPackage = "org.apache.streams.scala";
- String targetDirectory = "./target/generated-sources/scala-cli";
-
- StreamsScalaSourceGenerator.main(new String[]{sourcePackages, targetDirectory, targetPackage});
-
- File testOutput = new File(targetDirectory);
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists() );
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, scalaFilter, true);
- assert ( outputCollection.size() > 133 );
- }
-}
diff --git a/streams-plugins/streams-plugin-scala/src/test/java/org/apache/streams/plugins/test/StreamsScalaSourceGeneratorMojoIT.java b/streams-plugins/streams-plugin-scala/src/test/java/org/apache/streams/plugins/test/StreamsScalaSourceGeneratorMojoIT.java
deleted file mode 100644
index 84819fd..0000000
--- a/streams-plugins/streams-plugin-scala/src/test/java/org/apache/streams/plugins/test/StreamsScalaSourceGeneratorMojoIT.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.test;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.maven.it.Verifier;
-import org.apache.maven.it.util.ResourceExtractor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests that streams-plugin-pojo running via maven can convert activity schemas into pojos
- * which then compile.
- */
-public class StreamsScalaSourceGeneratorMojoIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsScalaSourceGeneratorMojoIT.class);
-
-// protected void setUp() throws Exception {
-// // required for mojo lookups to work
-// super.setUp();
-// }
-
- @Test
- public void testStreamsScalaSourceGeneratorMojo() throws Exception {
-
- File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-scala" );
-
- Verifier verifier;
-
- verifier = new Verifier( testDir.getAbsolutePath() );
-
- List<String> cliOptions = new ArrayList<>();
- cliOptions.add( "-N" );
- verifier.executeGoals(Stream.of(
- "compile").collect(Collectors.toList()));
-
- verifier.verifyErrorFreeLog();
-
- verifier.resetStreams();
-
- File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-sources/scala-mojo");
-
- Assert.assertNotNull(testOutput);
- Assert.assertTrue(testOutput.exists());
- Assert.assertTrue(testOutput.isDirectory());
-
- Collection<File> outputCollection = FileUtils.listFiles(testOutput, StreamsScalaSourceGeneratorTest.scalaFilter, true);
- assert ( outputCollection.size() > 133 );
-
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-scala/src/test/java/org/apache/streams/plugins/test/StreamsScalaSourceGeneratorTest.java b/streams-plugins/streams-plugin-scala/src/test/java/org/apache/streams/plugins/test/StreamsScalaSourceGeneratorTest.java
deleted file mode 100644
index 406099c..0000000
--- a/streams-plugins/streams-plugin-scala/src/test/java/org/apache/streams/plugins/test/StreamsScalaSourceGeneratorTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.plugins.test;
-
-import org.apache.streams.plugins.StreamsScalaGenerationConfig;
-import org.apache.streams.plugins.StreamsScalaSourceGenerator;
-
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests that StreamsScalaSourceGenerator via SDK generates scala sources.
- */
-public class StreamsScalaSourceGeneratorTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsScalaSourceGeneratorTest.class);
-
- public static final String[] scalaFilter = new String[]{"scala"};
-
- /**
- * Tests that StreamsScalaSourceGenerator via SDK generates scala sources.
- *
- * @throws Exception Exception
- */
- @Test
- public void testStreamsScalaSourceGenerator() throws Exception {
-
- StreamsScalaGenerationConfig streamsScalaGenerationConfig = new StreamsScalaGenerationConfig();
- streamsScalaGenerationConfig.setSourcePackages(Stream.of("org.apache.streams.pojo.json").collect(Collectors.toList()));
- streamsScalaGenerationConfig.setTargetPackage("org.apache.streams.scala");
- streamsScalaGenerationConfig.setTargetDirectory("target/generated-sources/scala-test");
-
- StreamsScalaSourceGenerator streamsScalaSourceGenerator = new StreamsScalaSourceGenerator(streamsScalaGenerationConfig);
- streamsScalaSourceGenerator.run();
-
- File testOutput = new File( "./target/generated-sources/scala-test/org/apache/streams/scala");
- FileFilter scalaFilter = pathname -> pathname.getName().endsWith(".scala");
-
- assertNotNull( testOutput );
- assertTrue( testOutput.exists() );
- assertTrue( testOutput.isDirectory() );
- assertEquals( 10, testOutput.listFiles(scalaFilter).length );
- assertTrue( new File(testOutput + "/traits").exists() );
- assertTrue( new File(testOutput + "/traits").isDirectory() );
- assertNotNull( new File(testOutput + "/traits").listFiles(scalaFilter) );
- assertEquals( 4, new File(testOutput + "/traits").listFiles(scalaFilter).length );
- assertTrue( new File(testOutput + "/objectTypes").exists() );
- assertTrue( new File(testOutput + "/objectTypes").isDirectory() );
- assertNotNull( new File(testOutput + "/objectTypes").listFiles(scalaFilter) );
- assertEquals( 42, new File(testOutput + "/objectTypes").listFiles(scalaFilter).length);
- assertTrue( new File(testOutput + "/verbs").exists() );
- assertTrue( new File(testOutput + "/verbs").isDirectory() );
- assertNotNull( new File(testOutput + "/verbs").listFiles(scalaFilter) );
- assertEquals( 89, new File(testOutput + "/verbs").listFiles(scalaFilter).length );
- }
-}
\ No newline at end of file
diff --git a/streams-plugins/streams-plugin-scala/src/test/resources/streams-plugin-scala/pom.xml b/streams-plugins/streams-plugin-scala/src/test/resources/streams-plugin-scala/pom.xml
deleted file mode 100644
index fb03784..0000000
--- a/streams-plugins/streams-plugin-scala/src/test/resources/streams-plugin-scala/pom.xml
+++ /dev/null
@@ -1,117 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-scala-test</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>Test StreamsPojoScalaMojo</name>
-
- <repositories>
- <repository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- </repositories>
-
- <pluginRepositories>
- <pluginRepository>
- <id>apache-repo</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/releases</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </pluginRepository>
- <pluginRepository>
- <id>apache-snapshots</id>
- <name>Apache Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </pluginRepository>
- </pluginRepositories>
-
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-scala</artifactId>
- <version>0.5-incubating-SNAPSHOT</version>
- <configuration>
- <packages>org.apache.streams.pojo.json</packages>
- <target>target/generated-sources/scala-mojo</target>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>scala</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-pojo-extensions/README.md b/streams-pojo-extensions/README.md
deleted file mode 100644
index e666093..0000000
--- a/streams-pojo-extensions/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-pojo-extensions
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-pojo-extensions/pom.xml b/streams-pojo-extensions/pom.xml
deleted file mode 100644
index 38f5432..0000000
--- a/streams-pojo-extensions/pom.xml
+++ /dev/null
@@ -1,67 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>apache-streams</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>streams-pojo-extensions</artifactId>
-
- <name>streams-pojo-extensions</name>
-
- <description>Support for adding and modifying extension properties</description>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
-
- </build>
-</project>
diff --git a/streams-pojo-extensions/src/main/java/org/apache/streams/pojo/extensions/ExtensionUtil.java b/streams-pojo-extensions/src/main/java/org/apache/streams/pojo/extensions/ExtensionUtil.java
deleted file mode 100644
index 2c6555b..0000000
--- a/streams-pojo-extensions/src/main/java/org/apache/streams/pojo/extensions/ExtensionUtil.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.pojo.extensions;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang3.StringUtils;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Class makes it easier to manage extensions added to activities, actors, objects, etc...
- */
-public class ExtensionUtil {
-
- public static final String DEFAULT_EXTENSION_PROPERTY = null;
-
- private static final ExtensionUtil INSTANCE = new ExtensionUtil(DEFAULT_EXTENSION_PROPERTY);
-
- private String extensionProperty;
-
- public static ExtensionUtil getInstance() {
- return INSTANCE;
- }
-
- public static ExtensionUtil getInstance(String property) {
- return new ExtensionUtil(property);
- }
-
- private ExtensionUtil(String extensionProperty) {
- this.extensionProperty = extensionProperty;
- }
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- public Map<String, Object> getExtensions(Activity activity) {
- return ensureExtensions(activity);
- }
-
- public Map<String, Object> getExtensions(ActivityObject object) {
- ActivityObject activityObject = mapper.convertValue(object, ActivityObject.class);
- return ensureExtensions(activityObject);
- }
-
- public Object getExtension(Activity activity, String key) {
- Map<String,Object> extensions = ensureExtensions(activity);
- return extensions.get(key);
- }
-
- public Object getExtension(ActivityObject object, String key) {
- Map<String,Object> extensions = ensureExtensions(object);
- return extensions.get(key);
- }
-
- public void addExtension(Activity activity, String key, Object extension) {
- Map<String,Object> extensions = ensureExtensions(activity);
- extensions.put(key, extension);
- }
-
- public void addExtension(ActivityObject object, String key, Object extension) {
- Map<String,Object> extensions = ensureExtensions(object);
- extensions.put(key, extension);
- }
-
- public void removeExtension(Activity activity, String key) {
- Map<String,Object> extensions = ensureExtensions(activity);
- extensions.remove(key);
- }
-
- public void removeExtension(ActivityObject object, String key) {
- Map<String,Object> extensions = ensureExtensions(object);
- extensions.remove(key);
- }
-
- public void setExtensions(Activity activity, Map<String, Object> extensions) {
- activity.setAdditionalProperty(extensionProperty, extensions);
- }
-
- public void setExtensions(ActivityObject object, Map<String, Object> extensions) {
- object.setAdditionalProperty(extensionProperty, extensions);
- }
-
- public void addExtensions(Activity activity, Map<String, Object> extensions) {
- for ( Map.Entry<String, Object> item : extensions.entrySet()) {
- addExtension(activity, item.getKey(), item.getValue());
- }
- }
-
- public void addExtensions(ActivityObject object, Map<String, Object> extensions) {
- for ( Map.Entry<String, Object> item : extensions.entrySet()) {
- addExtension(object, item.getKey(), item.getValue());
- }
- }
-
- /**
- * Creates a standard extension property.
- * @param activity activity to create the property in
- * @return the Map representing the extensions property
- */
- @SuppressWarnings("unchecked")
- public Map<String, Object> ensureExtensions(Activity activity) {
- Map<String,Object> additionalProperties = activity.getAdditionalProperties();
- Map<String,Object> extensions;
- if (additionalProperties == null) {
- additionalProperties = new HashMap<>();
- }
- if (StringUtils.isNotBlank(extensionProperty) ) {
- extensions = (Map<String, Object>) additionalProperties.get(extensionProperty);
- if (extensions == null) {
- extensions = new HashMap<>();
- additionalProperties.put(extensionProperty, extensions);
- }
- return extensions;
- } else {
- return additionalProperties;
- }
- }
-
- /**
- * Creates a standard extension property.
- * @param object object node to create the property in
- * @return {@link Map} representing the extensions property
- */
- @SuppressWarnings("unchecked")
- public Map<String, Object> ensureExtensions(ActivityObject object) {
- Map<String,Object> additionalProperties = object.getAdditionalProperties();
- Map<String,Object> extensions;
- if (additionalProperties == null) {
- additionalProperties = new HashMap<>();
- }
- if (StringUtils.isNotBlank(extensionProperty) ) {
- extensions = (Map<String, Object>) additionalProperties.get(extensionProperty);
- if (extensions == null) {
- extensions = new HashMap<>();
- additionalProperties.put(extensionProperty, extensions);
- }
- return extensions;
- } else {
- return additionalProperties;
- }
- }
-
-}
diff --git a/streams-pojo-extensions/src/main/jsonschema/activity.json b/streams-pojo-extensions/src/main/jsonschema/activity.json
deleted file mode 100644
index 39caadc..0000000
--- a/streams-pojo-extensions/src/main/jsonschema/activity.json
+++ /dev/null
@@ -1,51 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "title": "activity",
- "javaType": "org.apache.streams.pojo.extensions.Activity",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {
- "$ref": "../../../../streams-pojo/src/main/jsonschema/activity.json"
- },
- "description": "Activity with Apache Streams extended fields.",
- "properties": {
- "extensions": {
- "javaType": "org.apache.streams.pojo.extensions.ActivityExtensions",
- "javaInterfaces": ["java.io.Serializable"],
- "type": "object",
- "properties": {
- "likes": {
- "type": "integer",
- "description": "The number of +1, Like, favorites, etc that the post has received"
- },
- "rebroadcasts": {
- "type": "integer",
- "description": "The number of retweets, shares, etc that the post has received"
- },
- "language": {
- "type": "string",
- "description": "The ISO language code of the post"
- },
- "location": {
- "type": "object",
- "description": "Location that the post was made",
- "properties": {
- "country": {
- "type": "string",
- "description": "Country"
- },
- "coordinates": {
- "type": "array",
- "items": "number",
- "description": "Specific Geo-JSON coordinates (long,lat)"
- }
- }
- }
- }
- }
- }
-}
diff --git a/streams-pojo-extensions/src/main/jsonschema/object.json b/streams-pojo-extensions/src/main/jsonschema/object.json
deleted file mode 100644
index 01723d1..0000000
--- a/streams-pojo-extensions/src/main/jsonschema/object.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "title": "activity",
- "javaType": "org.apache.streams.pojo.extensions.ActivityObject",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {
- "$ref": "../../../../streams-pojo/src/main/jsonschema/object.json"
- },
- "description": "ActivityObject with Apache Streams extended fields.",
- "properties": {
- "extensions": {
- "javaType": "org.apache.streams.pojo.extensions.ObjectExtensions",
- "javaInterfaces": ["java.io.Serializable"],
- "type": "object",
- "properties": {
- "followers": {
- "type": "integer",
- "description": "The number of followers that the actor/object has"
- },
- "friends": {
- "type": "integer",
- "description": "The number of followers that the actor/object has"
- },
- "handle": {
- "type": "string",
- "description": "Screen name of the actor/object, distinct from displayName"
- },
- "language": {
- "type": "string",
- "description": "The ISO language code the actor/object prefers"
- },
- "likes": {
- "type": "integer",
- "description": "The number of likes that the actor/object has"
- },
- "location": {
- "type": "object",
- "description": "Location from which the actor/object most frequently posts",
- "properties": {
- "country": {
- "type": "string",
- "description": "Country"
- },
- "coordinates": {
- "type": "array",
- "items": "number",
- "description": "Specific Geo-JSON coordinates (long,lat)"
- }
- }
- },
- "posts": {
- "type": "integer",
- "description": "The number of posts that the actor/object has"
- }
- }
- }
- }
-}
diff --git a/streams-pojo-extensions/src/site/markdown/README.md b/streams-pojo-extensions/src/site/markdown/README.md
deleted file mode 100644
index 9f43db7..0000000
--- a/streams-pojo-extensions/src/site/markdown/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-streams-pojo-extensions
-==============
-
-Support for adding and modifying extension properties.
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-pojo-extensions/src/test/java/org/apache/streams/pojo/extensions/test/ExtensionUtilTest.java b/streams-pojo-extensions/src/test/java/org/apache/streams/pojo/extensions/test/ExtensionUtilTest.java
deleted file mode 100644
index b63220f..0000000
--- a/streams-pojo-extensions/src/test/java/org/apache/streams/pojo/extensions/test/ExtensionUtilTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.pojo.extensions.test;
-
-import org.apache.streams.pojo.extensions.ExtensionUtil;
-import org.apache.streams.pojo.json.Activity;
-
-import org.apache.commons.lang3.StringUtils;
-import org.junit.Test;
-
-import java.util.Map;
-
-/**
- * Test ExtensionUtil methods.
- */
-public class ExtensionUtilTest {
-
- @Test
- public void testActivitySetCustomExtension() throws Exception {
- ExtensionUtil customExtensionUtil = ExtensionUtil.getInstance("ext");
- Activity activity = new Activity();
- Map<String, Object> extensions = customExtensionUtil.ensureExtensions(activity);
- String value = "value";
- extensions.put("extension", value);
- customExtensionUtil.setExtensions(activity, extensions);
- assert (StringUtils.isNotBlank((String) customExtensionUtil.getExtension(activity, "extension")));
- extensions = customExtensionUtil.getExtensions(activity);
- assert (value.equals((String)extensions.get("extension")));
- assert (activity.getAdditionalProperties().get("ext") != null);
- }
-
-}
diff --git a/streams-pojo/README.md b/streams-pojo/README.md
deleted file mode 100644
index 7418045..0000000
--- a/streams-pojo/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-pojo
-===========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-pojo/metadata/Serializer.mup b/streams-pojo/metadata/Serializer.mup
deleted file mode 100644
index 8552b3b..0000000
--- a/streams-pojo/metadata/Serializer.mup
+++ /dev/null
@@ -1,374 +0,0 @@
-{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "title": "Serializer",
- "id": 1,
- "formatVersion": 2,
- "attr": {
- "style": {}
- },
- "ideas": {
- "1": {
- "title": "Activity",
- "id": 2,
- "ideas": {
- "1": {
- "title": "actor",
- "id": 4,
- "ideas": {
- "1": {
- "title": "summary",
- "id": 28
- },
- "7": {
- "title": "url",
- "id": 41
- },
- "0.5": {
- "title": "displayName",
- "id": 32,
- "attr": {
- "style": {}
- }
- },
- "0.75": {
- "title": "extensions",
- "id": 33,
- "ideas": {
- "1": {
- "title": "followers",
- "id": 34
- },
- "2": {
- "title": "handle",
- "id": 35
- },
- "3": {
- "title": "location",
- "id": 36
- },
- "4": {
- "title": "location",
- "id": 37
- },
- "5": {
- "title": "screenName",
- "id": 38
- },
- "6": {
- "title": "posts",
- "id": 39
- },
- "7": {
- "title": "favorites",
- "id": 40
- }
- },
- "attr": {
- "style": {}
- }
- },
- "0.875": {
- "title": "handle",
- "id": 30,
- "attr": {
- "style": {}
- }
- },
- "0.9375": {
- "title": "id",
- "id": 29,
- "attr": {
- "style": {}
- }
- },
- "0.96875": {
- "title": "image",
- "id": 31,
- "attr": {
- "style": {}
- }
- }
- }
- },
- "2": {
- "title": "id",
- "id": 5
- },
- "3": {
- "title": "verb",
- "id": 6
- },
- "1.5": {
- "title": "content",
- "id": 16,
- "attr": {
- "style": {}
- }
- },
- "2.5": {
- "title": "icon",
- "id": 13,
- "attr": {
- "style": {}
- }
- },
- "1.75": {
- "title": "generator",
- "id": 12,
- "attr": {
- "style": {}
- },
- "ideas": {
- "1": {
- "title": "id",
- "id": 62
- },
- "0.5": {
- "title": "displayName",
- "id": 63,
- "attr": {
- "style": {}
- }
- }
- }
- },
- "2.75": {
- "title": "links",
- "id": 18,
- "attr": {
- "style": {}
- }
- },
- "2.875": {
- "title": "object",
- "id": 8,
- "attr": {
- "style": {}
- },
- "ideas": {
- "1": {
- "title": "content",
- "id": 22
- },
- "2": {
- "title": "id",
- "id": 23
- },
- "4": {
- "title": "objectType",
- "id": 25
- },
- "1.5": {
- "title": "geo",
- "id": 24,
- "ideas": {
- "1": {
- "title": "coordinates",
- "id": 26
- }
- },
- "attr": {
- "style": {}
- }
- }
- }
- },
- "2.9375": {
- "title": "published",
- "id": 10,
- "attr": {
- "style": {}
- },
- "ideas": {}
- },
- "2.90625": {
- "title": "provider",
- "id": 14,
- "attr": {
- "style": {}
- },
- "ideas": {
- "1": {
- "title": "id",
- "id": 20
- },
- "0.5": {
- "title": "displayName",
- "id": 21,
- "attr": {
- "style": {}
- }
- }
- }
- },
- "2.96875": {
- "title": "target",
- "id": 9,
- "attr": {
- "style": {}
- },
- "ideas": {
- "1": {
- "title": "id",
- "id": 64
- },
- "2": {
- "title": "image",
- "id": 65
- },
- "4": {
- "title": "summary",
- "id": 67
- },
- "0.5": {
- "title": "displayName",
- "id": 66,
- "attr": {
- "style": {}
- }
- },
- "0.25": {
- "title": "content",
- "id": 68,
- "attr": {
- "style": {}
- }
- }
- }
- },
- "2.984375": {
- "title": "updated",
- "id": 11,
- "attr": {
- "style": {}
- }
- },
- "2.9765625": {
- "title": "title",
- "id": 15,
- "attr": {
- "style": {}
- }
- },
- "2.9921875": {
- "title": "url",
- "id": 17,
- "attr": {
- "style": {}
- }
- },
- "1.625": {
- "title": "extensions",
- "id": 42,
- "ideas": {
- "1": {
- "title": "source",
- "id": 43,
- "ideas": {
- "1": {
- "title": "source json",
- "id": 44
- }
- }
- },
- "6": {
- "title": "user_mentions",
- "id": 58,
- "ideas": {
- "1": {
- "title": "id",
- "id": 59
- },
- "2": {
- "title": "handle",
- "id": 60
- },
- "3": {
- "title": "displayName",
- "id": 61
- }
- }
- },
- "0.5": {
- "title": "hashtags",
- "id": 57,
- "attr": {
- "style": {}
- }
- },
- "0.75": {
- "title": "location",
- "id": 48,
- "ideas": {
- "1": {
- "title": "id",
- "id": 49
- },
- "2": {
- "title": "coordinates",
- "id": 50,
- "ideas": {
- "1": {
- "title": "type",
- "id": 52
- },
- "2": {
- "title": "coordinates",
- "id": 53
- }
- }
- }
- },
- "attr": {
- "style": {}
- }
- },
- "0.625": {
- "title": "likes",
- "id": 54,
- "ideas": {
- "1": {
- "title": "perspectival",
- "id": 55
- },
- "2": {
- "title": "count",
- "id": 56
- }
- },
- "attr": {
- "style": {}
- }
- },
- "0.875": {
- "title": "rebroadcasts",
- "id": 45,
- "ideas": {
- "1": {
- "title": "perspectival",
- "id": 46
- },
- "2": {
- "title": "count",
- "id": 47
- }
- },
- "attr": {
- "style": {}
- }
- }
- },
- "attr": {
- "style": {}
- }
- }
- }
- },
- "-1": {
- "title": "Source Data",
- "id": 3
- }
- }
-}
\ No newline at end of file
diff --git a/streams-pojo/metadata/serializer.png b/streams-pojo/metadata/serializer.png
deleted file mode 100644
index 3f99b8e..0000000
Binary files a/streams-pojo/metadata/serializer.png and /dev/null differ
diff --git a/streams-pojo/pom.xml b/streams-pojo/pom.xml
deleted file mode 100644
index d911a55..0000000
--- a/streams-pojo/pom.xml
+++ /dev/null
@@ -1,264 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>apache-streams</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>streams-pojo</artifactId>
-
- <name>streams-pojo</name>
-
- <description>Activity json schemas and POJOs, and supporting classes.</description>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.juneau</groupId>
- <artifactId>juneau-marshall</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-joda</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_${scala.suffix}</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-xml</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml</groupId>
- <artifactId>jackson-xml-databind</artifactId>
- <version>0.6.2</version>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml</groupId>
- <artifactId>aalto-xml</artifactId>
- </dependency>
-
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
-
- <dependency>
- <groupId>nz.net.ultraq.jaxb</groupId>
- <artifactId>jaxb-utilities</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-validator</groupId>
- <artifactId>commons-validator</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>javax.validation</groupId>
- <artifactId>validation-api</artifactId>
- <version>1.1.0.Final</version>
- </dependency>
-
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- <version>${reflections.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
-
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- <version>${logback.version}</version>
- </dependency>
-
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-core</artifactId>
- <version>${logback.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>unpack-schemas</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- </configuration>
- </execution>
- <execution>
- <id>unpack-testdocs</id>
- <phase>process-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-testdocs</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>test-jar</includeTypes>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- <sourcePath>${project.basedir}/target/test-classes/activitystreams-schemas</sourcePath>
- <sourcePath>${project.basedir}/target/test-classes/activitystreams-schemas/objectTypes</sourcePath>
- <sourcePath>${project.basedir}/target/test-classes/activitystreams-schemas/verbs</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.pojo.json</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <includes>
- <include>**/*.conf</include>
- <include>**/*.json</include>
- <include>**/*.class</include>
- </includes>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-pojo/src/main/java/org/apache/streams/data/ActivityConverter.java b/streams-pojo/src/main/java/org/apache/streams/data/ActivityConverter.java
deleted file mode 100644
index 70f2135..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/data/ActivityConverter.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.data;
-
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.Activity;
-
-import java.io.Serializable;
-import java.util.List;
-
-/**
- * Converts non-Activity documents to Activities and back.
- *
- * <p/>
- * Each converter may one, several, or zero activities.
- *
- * <p/>
- * The recommended approach for deriving multiple activities from a source document is:
- *
- * <p/>
- * 1) Return one activity for each occurance of a verb, from the same ActivityConverter, if the activities are of like type.
- *
- * <p/>
- * For example, BlogShareConverter would convert a blog containing two shares into two Activities with verb: share
- *
- * <p/>
- * 2) Create multiple ActivityConverters, if the activities are not of like type.
- *
- * <p/>
- * For example, a blog post that is both a post and a share should be transformed by two seperate Converters, individually
- * or simultaneously applied.
- */
-public interface ActivityConverter<T> extends Serializable {
-
- /**
- * What class does this ActivityConverter require?
- *
- * @return The class the ActivityConverter requires. Should always return the templated class.
- */
- Class requiredClass();
-
- /**
- * Gets the supported content type that can be deserialized/serialized
- *
- * @return A string representing the format name. Can be an IETF MIME type or other
- */
- String serializationFormat();
-
- /**
- * Converts the activity to a POJO representation.
- *
- * @param deserialized the string
- * @return a fully populated Activity object
- */
- T fromActivity(Activity deserialized) throws ActivityConversionException;
-
- /**
- * Converts multiple Activities into a list of source documents.
- * @param list a typed List of documents
- * @return a list of source documents
- */
- List<T> fromActivityList(List<Activity> list) throws ActivityConversionException;
-
- /**
- * Converts a POJO into one or more Activities.
- * @param serialized the string representation
- * @return a fully populated Activity object
- */
- List<Activity> toActivityList(T serialized) throws ActivityConversionException;
-
- /**
- * Converts multiple documents into a list of Activity objects.
- * @param list a typed List of documents
- * @return a list of fully populated activities
- */
- List<Activity> toActivityList(List<T> list) throws ActivityConversionException;
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/data/ActivityObjectConverter.java b/streams-pojo/src/main/java/org/apache/streams/data/ActivityObjectConverter.java
deleted file mode 100644
index 7e80327..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/data/ActivityObjectConverter.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.data;
-
-import org.apache.streams.exceptions.ActivityConversionException;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import java.io.Serializable;
-
-/**
- * Converts non-ActivityObject documents to ActivityObjects and back.
- *
- * <p/>
- * Each converter may return zero or one alternative representations.
- */
-public interface ActivityObjectConverter<T> extends Serializable {
-
- /**
- * What class does this ActivityConverter require?
- *
- * @return The class the ActivityConverter requires. Should always return the templated class.
- */
- Class requiredClass();
-
- /**
- * Gets the supported content type that can be deserialized/serialized.
- *
- * @return A string representing the format name. Can be an IETF MIME type or other
- */
- String serializationFormat();
-
- /**
- * Converts the activity to a POJO representation.
- *
- * @param deserialized the string
- * @return a fully populated Activity object
- */
- T fromActivityObject(ActivityObject deserialized) throws ActivityConversionException;
-
- /**
- * Converts a POJO into an ActivityObject.
- * @param serialized the string representation
- * @return a fully populated Activity object
- */
- ActivityObject toActivityObject(T serialized) throws ActivityConversionException;
-
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/data/ActivitySerializer.java b/streams-pojo/src/main/java/org/apache/streams/data/ActivitySerializer.java
deleted file mode 100644
index 3a4282c..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/data/ActivitySerializer.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.data;
-
-import org.apache.streams.exceptions.ActivitySerializerException;
-import org.apache.streams.pojo.json.Activity;
-
-import java.util.List;
-
-/**
- * Serializes and deserializes Activities
- *
- * <p/>
- * Deprecated: Switch all modules to use {@link org.apache.streams.data.ActivityConverter}
- */
-@Deprecated
-public interface ActivitySerializer<T> {
-
- /**
- * Gets the supported content type that can be deserialized/serialized.
- *
- * @return A string representing the format name. Can be an IETF MIME type or other
- */
- String serializationFormat();
-
- /**
- * Converts the activity to a POJO representation.
- *
- * @param deserialized the string
- * @return a fully populated Activity object
- */
- T serialize(Activity deserialized) throws ActivitySerializerException;
-
- /**
- * Converts a POJO into an Activity.
- * @param serialized the string representation
- * @return a fully populated Activity object
- */
- Activity deserialize(T serialized) throws ActivitySerializerException;
-
- /**
- * Converts multiple documents into a list of Activity objects.
- * @param serializedList a typed List of documents
- * @return a list of fully populated activities
- */
- List<Activity> deserializeAll(List<T> serializedList);
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/data/DocumentClassifier.java b/streams-pojo/src/main/java/org/apache/streams/data/DocumentClassifier.java
deleted file mode 100644
index b15722f..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/data/DocumentClassifier.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.data;
-
-import java.io.Serializable;
-import java.util.List;
-
-/**
- * DocumentClassifier assists with ActivityConversion, by determining whether a document may be
- * parseable into a POJO for which an ActivityConverter exists.
- */
-public interface DocumentClassifier extends Serializable {
-
- /**
- * Assess the structure of the document, and identify whether the provided document is
- * a structural match for one or more typed forms.
- *
- * @param document the document
- * @return a serializable pojo class this document matches
- */
- List<Class> detectClasses(Object document);
-
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java b/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java
deleted file mode 100644
index c7e8337..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/data/util/ActivityUtil.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.data.util;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Utility class for managing activities
- *
- * <p/>
- * Deprecated: Use {@link org.apache.streams.pojo.extensions.ExtensionUtil}
- */
-@Deprecated
-public class ActivityUtil {
-
- private ActivityUtil() {}
-
- /**
- * Property on the activity object to use for extensions.
- */
- public static final String EXTENSION_PROPERTY = "extensions";
-
- /**
- * The number of +1, Like, favorites, etc that the post has received.
- */
- public static final String LIKES_EXTENSION = "likes";
-
- /**
- * The number of retweets, shares, etc that the post has received.
- */
- public static final String REBROADCAST_EXTENSION = "rebroadcasts";
-
- /**
- * The language of the post.
- */
- public static final String LANGUAGE_EXTENSION = "language";
-
- /**
- * Location that the post was made or the actor's residence.
- */
- public static final String LOCATION_EXTENSION = "location";
-
- /**
- * Country that the post was made.
- */
- public static final String LOCATION_EXTENSION_COUNTRY = "country";
-
- /**
- * Specific JSON-geo coordinates (long,lat).
- */
- public static final String LOCATION_EXTENSION_COORDINATES = "coordinates";
-
- private static final ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- /**
- * Creates a standard extension property.
- * @param activity activity to create the property in
- * @return the Map representing the extensions property
- */
- @SuppressWarnings("unchecked")
- @Deprecated
- public static Map<String, Object> ensureExtensions(Activity activity) {
- Map<String, Object> extensions = (Map)activity.getAdditionalProperties().get(EXTENSION_PROPERTY);
- if (extensions == null) {
- extensions = new HashMap<>();
- activity.setAdditionalProperty(EXTENSION_PROPERTY, extensions);
- }
- return extensions;
- }
-
- /**
- * Gets a formatted ID.
- * @param providerName name of the provider
- * @param personId ID of the person within the system
- * @return id:{providerName}:people:{personId}
- */
- public static String getPersonId(String providerName, String personId) {
- return String.format("id:%s:people:%s", providerName, personId);
- }
-
- /**
- * Gets a formatted provider ID.
- * @param providerName name of the provider
- * @return id:providers:{providerName}
- */
- public static String getProviderId(String providerName) {
- return String.format("id:providers:%s", providerName);
- }
-
- /**
- * Gets a formatted object ID.
- * @param provider name of the provider
- * @param objectType type of the object
- * @param objectId the ID of the object
- * @return id:{provider}:{objectType}s:{objectId}
- */
- public static String getObjectId(String provider, String objectType, String objectId) {
- return String.format("id:%s:%ss:%s", provider, objectType, objectId);
- }
-
- /**
- * Gets a formatted activity ID.
- * @param providerName name of the provider
- * @param activityId ID of the provider
- * @return id:{providerName}:activities:{activityId}
- */
- public static String getActivityId(String providerName, String activityId) {
- return String.format("id:%s:activities:%s", providerName, activityId);
- }
-
- /**
- * Check validity of Activity.
- * @param activity Activity
- * @return isValid
- */
- public static boolean isValid(Activity activity) {
- return activity != null
- && activity.getId() != null
- && activity.getVerb() != null
- && activity.getProvider() != null
- && activity.getProvider().getId() != null;
- }
-
- /**
- * Check validity of ActivityObject.
- * @param activityObject ActivityObject
- * @return isValid
- */
- public static boolean isValid(ActivityObject activityObject) {
- return activityObject != null
- && activityObject.getId() != null
- && activityObject.getObjectType() != null;
- }
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/data/util/RFC3339Utils.java b/streams-pojo/src/main/java/org/apache/streams/data/util/RFC3339Utils.java
deleted file mode 100644
index 18ae064..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/data/util/RFC3339Utils.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-
-package org.apache.streams.data.util;
-
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.DateTimeFormatterBuilder;
-import org.joda.time.format.DateTimeParser;
-
-import java.util.TimeZone;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Parses and formats dates to Joda Time {@link org.joda.time.DateTime} and to RFC3339 compatible Strings
- */
-public class RFC3339Utils {
-
- private static final RFC3339Utils INSTANCE = new RFC3339Utils();
-
- public static RFC3339Utils getInstance() {
- return INSTANCE;
- }
-
- private static final String BASE = "^[0-9]{4}\\-[0-9]{2}\\-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}";
- private static final String TZ = "[+-][0-9]{2}:?[0-9]{2}$";
- private static final String SUB_SECOND = "\\.([0-9]*)";
- private static final String UTC = "Z$";
-
-
- private static final Pattern MILLIS = Pattern.compile("^[0-9]*$");
- private static final Pattern UTC_STANDARD = Pattern.compile(BASE + UTC);
- private static final Pattern UTC_SUB_SECOND = Pattern.compile(BASE + SUB_SECOND + UTC);
- private static final Pattern LOCAL_STANDARD = Pattern.compile(BASE + TZ);
- private static final Pattern LOCAL_SUB_SECOND = Pattern.compile(BASE + SUB_SECOND + TZ);
-
- private static final String BASE_FMT = "yyyy-MM-dd'T'HH:mm:ss";
- public static final DateTimeFormatter UTC_STANDARD_FMT = DateTimeFormat.forPattern(BASE_FMT + "'Z'").withZoneUTC();
- public static final DateTimeFormatter UTC_SUB_SECOND_FMT = DateTimeFormat.forPattern(BASE_FMT + ".SSS'Z'").withZoneUTC();
- public static final DateTimeFormatter LOCAL_STANDARD_FMT = DateTimeFormat.forPattern(BASE_FMT + "Z").withZoneUTC();
- public static final DateTimeFormatter LOCAL_SUB_SECOND_FMT = DateTimeFormat.forPattern(BASE_FMT + ".SSSZ").withZoneUTC();
-
- /**
- * Contains various formats. All formats should be of international standards when comes to the ordering of the
- * days and month.
- */
- private static final DateTimeFormatter DEFAULT_FORMATTER;
- /**
- * Contains alternative formats that will succeed after failures from the DEFAULT_FORMATTER.
- * i.e. 4/24/2014 will throw an exception on the default formatter because it will assume international date standards
- * However, the date will parse in the ALT_FORMATTER because it contains the US format of MM/dd/yyyy.
- */
- private static final DateTimeFormatter ALT_FORMATTER;
-
- static {
- DateTimeParser[] parsers = new DateTimeParser[]{
- DateTimeFormat.forPattern("EEE MMM dd HH:mm:ss Z yyyy").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("EEE, dd MMM yyyy HH:mm:ss Z").getParser(),
- DateTimeFormat.forPattern("dd MMMM yyyy HH:mm:ss").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("yyyyMMdd").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("dd-MM-yyyy").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("yyyy-MM-dd").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("yyyy/MM/dd").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("dd MMM yyyy").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("dd MMMM yyyy").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("yyyyMMddHHmm").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("yyyyMMdd HHmm").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("dd-MM-yyyy HH:mm").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("yyyy-MM-dd HH:mm").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("yyyy/MM/dd HH:mm").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("dd MMM yyyy HH:mm").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("dd MMMM yyyy HH:mm").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("yyyyMMddHHmmss").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("yyyyMMdd HHmmss").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("dd-MM-yyyy HH:mm:ss").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("yyyy/MM/dd HH:mm:ss").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("dd MMM yyyy HH:mm:ss").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("HH:mm:ss yyyy/MM/dd").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("HH:mm:ss MM/dd/yyyy").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("HH:mm:ss yyyy-MM-dd").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("HH:mm:ss MM-dd-yyyy").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("dd/MM/yyyy HH:mm:ss").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("dd/MM/yyyy HH:mm").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("dd/MM/yyyy").withZoneUTC().getParser(),
- UTC_STANDARD_FMT.getParser(),
- UTC_SUB_SECOND_FMT.getParser(),
- LOCAL_STANDARD_FMT.getParser(),
- LOCAL_SUB_SECOND_FMT.getParser()
- };
- DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
- builder.append(null, parsers);
- DEFAULT_FORMATTER = builder.toFormatter().withZoneUTC();
-
- DateTimeParser[] altParsers = new DateTimeParser[] {
- DateTimeFormat.forPattern("MM-dd-yyyy HH:mm:ss").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("MM/dd/yyyy HH:mm:ss").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("MM/dd/yyyy HH:mm").withZoneUTC().getParser(),
- DateTimeFormat.forPattern("MM/dd/yyyy").withZoneUTC().getParser(),
- };
- builder = new DateTimeFormatterBuilder();
- builder.append(null, altParsers);
- ALT_FORMATTER = builder.toFormatter().withZoneUTC();
- }
-
- private RFC3339Utils() {}
-
- /**
- * parse String to DateTime
- * @param toParse DateTime as UTC String
- * @return DateTime
- */
- public static DateTime parseUTC(String toParse) {
- if (MILLIS.matcher(toParse).matches()) {
- return new DateTime(Long.valueOf(toParse), DateTimeZone.UTC);
- }
- if (UTC_STANDARD.matcher(toParse).matches()) {
- return parseUTC(UTC_STANDARD_FMT, toParse);
- }
- Matcher utc = UTC_SUB_SECOND.matcher(toParse);
- if (utc.matches()) {
- return parseUTC(getSubSecondFormat(utc.group(1), "'Z'"), toParse);
- }
- if (LOCAL_STANDARD.matcher(toParse).matches()) {
- return parseUTC(LOCAL_STANDARD_FMT, toParse);
- }
- Matcher local = LOCAL_SUB_SECOND.matcher(toParse);
- if (local.matches()) {
- return parseUTC(getSubSecondFormat(local.group(1), "Z"), toParse);
- }
- throw new IllegalArgumentException(String.format("Failed to parse date %s. Ensure format is RFC3339 Compliant", toParse));
- }
-
- private static DateTime parseUTC(DateTimeFormatter formatter, String toParse) {
- return formatter.parseDateTime(toParse);
- }
-
- /**
- * Parses arbitrarily formatted Strings representing dates or dates and times to a {@link org.joda.time.DateTime}
- * objects. It first attempts parse with international standards, assuming the dates are either dd MM yyyy or
- * yyyy MM dd. If that fails it will try American formats where the month precedes the days of the month.
- * @param dateString abitrarily formatted date or date and time string
- * @return {@link org.joda.time.DateTime} representation of the dateString
- */
- public static DateTime parseToUTC(String dateString) {
- if (MILLIS.matcher(dateString).find()) {
- return new DateTime(Long.parseLong(dateString));
- }
- try {
- return DEFAULT_FORMATTER.parseDateTime(dateString);
- } catch (Exception ex) {
- return ALT_FORMATTER.parseDateTime(dateString);
- }
- }
-
- /**
- * Formats an arbitrarily formatted into RFC3339 Specifications.
- * @param dateString date string to be formatted
- * @return RFC3339 compliant date string
- */
- public static String format(String dateString) {
- return format(parseToUTC(dateString));
- }
-
- public static String format(DateTime toFormat) {
- return UTC_SUB_SECOND_FMT.print(toFormat.getMillis());
- }
-
- public static String format(DateTime toFormat, TimeZone tz) {
- return LOCAL_SUB_SECOND_FMT.withZone(DateTimeZone.forTimeZone(tz)).print(toFormat.getMillis());
- }
-
- private static DateTimeFormatter getSubSecondFormat(String sub, String suffix) {
- DateTimeFormatter result;
- //Since RFC3339 allows for any number of sub-second notations, we need to flexibly support more or less than 3
- //digits; however, if it is exactly 3, just use the standards.
- if (sub.length() == 3) {
- result = suffix.equals("Z") ? LOCAL_SUB_SECOND_FMT : UTC_SUB_SECOND_FMT;
- } else {
- StringBuilder pattern = new StringBuilder();
- pattern.append(BASE_FMT);
- pattern.append(".");
- for (int i = 0; i < sub.length(); i++) {
- pattern.append("S");
- }
- pattern.append(suffix);
- result = DateTimeFormat.forPattern(pattern.toString()).withZoneUTC();
- }
- return result;
- }
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/exceptions/ActivityConversionException.java b/streams-pojo/src/main/java/org/apache/streams/exceptions/ActivityConversionException.java
deleted file mode 100644
index 96c4bb1..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/exceptions/ActivityConversionException.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.exceptions;
-
-/**
- * ActivityConversionException is a typed exception appropriate when a valid Activity
- * cannot be created from a given document.
- */
-public class ActivityConversionException extends Exception {
-
- public ActivityConversionException() {
- }
-
- public ActivityConversionException(String message) {
- super(message);
- }
-
- public ActivityConversionException(Throwable cause) {
- super(cause);
- }
-
- public ActivityConversionException(String message, Throwable cause) {
- super(message, cause);
- }
-
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/exceptions/ActivitySerializerException.java b/streams-pojo/src/main/java/org/apache/streams/exceptions/ActivitySerializerException.java
deleted file mode 100644
index 85a3b8f..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/exceptions/ActivitySerializerException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.exceptions;
-
-/**
- * ActivitySerializerException is a typed exception appropriate when a valid Activity
- * cannot be created from a given document.
- */
-public class ActivitySerializerException extends Exception {
-
- public ActivitySerializerException() {
- // TODO Auto-generated constructor stub
- }
-
- public ActivitySerializerException(String message) {
- super(message);
- // TODO Auto-generated constructor stub
- }
-
- public ActivitySerializerException(Throwable cause) {
- super(cause);
- // TODO Auto-generated constructor stub
- }
-
- public ActivitySerializerException(String message, Throwable cause) {
- super(message, cause);
- // TODO Auto-generated constructor stub
- }
-
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
deleted file mode 100644
index 50ffb5d..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-import org.apache.streams.data.util.RFC3339Utils;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * StreamsDateTimeDeserializer is a supporting class for
- * @see {@link org.apache.streams.jackson.StreamsJacksonMapper}
- *
- * <p/>
- * Converting date-time strings other than RFC3339 to joda DateTime objects requires
- * additional formats to be provided when instantiating StreamsJacksonMapper.
- */
-public class StreamsDateTimeDeserializer extends StdDeserializer<DateTime> implements Serializable {
-
- private List<DateTimeFormatter> formatters = new ArrayList<>();
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsDateTimeDeserializer.class);
-
- protected StreamsDateTimeDeserializer(Class<DateTime> dateTimeClass) {
- super(dateTimeClass);
- }
-
- protected StreamsDateTimeDeserializer(Class<DateTime> dateTimeClass, List<String> formats) {
- super(dateTimeClass);
- for ( String format : formats ) {
- try {
- formatters.add(DateTimeFormat.forPattern(format));
- } catch (Exception ex) {
- LOGGER.warn("Exception parsing format " + format);
- }
- }
- }
-
- /**
- * Applies each additional format in turn, until it can provide a non-null DateTime
- */
- @Override
- public DateTime deserialize(JsonParser jpar, DeserializationContext context) throws IOException {
-
- DateTime result = RFC3339Utils.parseToUTC(jpar.getValueAsString());
- Iterator<DateTimeFormatter> iterator = formatters.iterator();
- while ( result == null && iterator.hasNext()) {
- DateTimeFormatter formatter = iterator.next();
- result = formatter.parseDateTime(jpar.getValueAsString());
- }
- return result;
- }
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeFormat.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeFormat.java
deleted file mode 100644
index 55b198d..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeFormat.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-/**
- * Supplies a custom date-time format to StreamsJacksonModule.
- *
- * <p/>
- * Implementations must have a no-argument constructor
- */
-public interface StreamsDateTimeFormat {
-
- String getFormat();
-
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
deleted file mode 100644
index 4310503..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-import org.apache.streams.data.util.RFC3339Utils;
-
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-import org.joda.time.DateTime;
-
-import java.io.IOException;
-import java.io.Serializable;
-
-/**
- * StreamsDateTimeSerializer is a supporting class for
- * @link org.apache.streams.jackson.StreamsJacksonMapper.
- */
-public class StreamsDateTimeSerializer extends StdSerializer<DateTime> implements Serializable {
-
- protected StreamsDateTimeSerializer(Class<DateTime> dateTimeClass) {
- super(dateTimeClass);
- }
-
- @Override
- public void serialize(DateTime value, JsonGenerator jgen, SerializerProvider provider) throws IOException {
- jgen.writeString(RFC3339Utils.format(value));
- }
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
deleted file mode 100644
index f31efc4..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonMapper.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-import org.apache.streams.pojo.StreamsJacksonMapperConfiguration;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.module.scala.DefaultScalaModule;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * StreamsJacksonMapper is the recommended interface to jackson for any streams component.
- *
- * <p/>
- * Date-time formats that must be supported can be specified with constructor arguments.
- *
- * <p/>
- * If no Date-time formats are specified, streams will use reflection to find formats.
- */
-public class StreamsJacksonMapper extends ObjectMapper {
-
- private static Map<StreamsJacksonMapperConfiguration, StreamsJacksonMapper> INSTANCE_MAP = new ConcurrentHashMap<>();
-
- private StreamsJacksonMapperConfiguration configuration = new StreamsJacksonMapperConfiguration();
-
- /**
- * get default StreamsJacksonMapper.
- * @return StreamsJacksonMapper
- */
- public static StreamsJacksonMapper getInstance() {
- return getInstance(new StreamsJacksonMapperConfiguration());
- }
-
- /**
- * get custom StreamsJacksonMapper.
- * @param configuration StreamsJacksonMapperConfiguration
- * @return StreamsJacksonMapper
- */
- public static StreamsJacksonMapper getInstance(StreamsJacksonMapperConfiguration configuration) {
- if ( INSTANCE_MAP.containsKey(configuration)
- &&
- INSTANCE_MAP.get(configuration) != null) {
- return INSTANCE_MAP.get(configuration);
- } else {
- INSTANCE_MAP.put(configuration, new StreamsJacksonMapper(configuration));
- return INSTANCE_MAP.get(configuration);
- }
- }
-
- /**
- * get custom StreamsJacksonMapper.
- * @param format format
- * @return StreamsJacksonMapper
- */
- @Deprecated
- public static StreamsJacksonMapper getInstance(String format) {
-
- return new StreamsJacksonMapper(Collections.singletonList(format));
-
- }
-
- /**
- * get custom StreamsJacksonMapper.
- * @param formats formats
- * @return StreamsJacksonMapper
- */
- @Deprecated
- public static StreamsJacksonMapper getInstance(List<String> formats) {
-
- return new StreamsJacksonMapper(formats);
-
- }
-
- /*
- Use getInstance to get a globally shared thread-safe ObjectMapper,
- rather than call this constructor. Reflection-based resolution of
- date-time formats across all modules can be slow and should only happen
- once per JVM.
- */
- protected StreamsJacksonMapper() {
- super();
- registerModule(new StreamsJacksonModule(configuration.getDateFormats()));
- if ( configuration.getEnableScala()) {
- registerModule(new DefaultScalaModule());
- }
- configure();
- }
-
- @Deprecated
- public StreamsJacksonMapper(String format) {
- super();
- registerModule(new StreamsJacksonModule(Collections.singletonList(format)));
- if ( configuration.getEnableScala()) {
- registerModule(new DefaultScalaModule());
- }
- configure();
- }
-
- @Deprecated
- public StreamsJacksonMapper(List<String> formats) {
- super();
- registerModule(new StreamsJacksonModule(formats));
- if ( configuration.getEnableScala()) {
- registerModule(new DefaultScalaModule());
- }
- configure();
- }
-
- public StreamsJacksonMapper(StreamsJacksonMapperConfiguration configuration) {
- super();
- registerModule(new StreamsJacksonModule(configuration.getDateFormats()));
- if ( configuration.getEnableScala()) {
- registerModule(new DefaultScalaModule());
- }
- configure();
- }
-
- public void configure() {
- disable(com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
- configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, Boolean.FALSE);
- configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, Boolean.TRUE);
- configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, Boolean.TRUE);
- configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, Boolean.TRUE);
- configure(DeserializationFeature.WRAP_EXCEPTIONS, Boolean.FALSE);
- configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, Boolean.TRUE);
- // If a user has an 'object' that does not have an explicit mapping, don't cause the serialization to fail.
- configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, Boolean.FALSE);
- configure(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS, Boolean.FALSE);
- configure(SerializationFeature.WRITE_NULL_MAP_VALUES, Boolean.FALSE);
- setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.DEFAULT);
- setSerializationInclusion(JsonInclude.Include.NON_EMPTY);
- }
-
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java
deleted file mode 100644
index 6445881..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsJacksonModule.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import org.joda.time.DateTime;
-import org.joda.time.Period;
-import org.reflections.Reflections;
-import org.reflections.scanners.SubTypesScanner;
-import org.reflections.util.ClasspathHelper;
-import org.reflections.util.ConfigurationBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-/**
- * StreamsJacksonModule is a supporting class for
- * @link org.apache.streams.jackson.StreamsJacksonMapper
- *
- * <p/>
- * RFC3339 dates are supported by default.
- */
-public class StreamsJacksonModule extends SimpleModule {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsJacksonModule.class);
-
- public StreamsJacksonModule() {
- super();
-
- Reflections reflections = new Reflections(new ConfigurationBuilder()
- .setUrls(ClasspathHelper.forPackage("org.apache.streams.jackson"))
- .setScanners(new SubTypesScanner()));
-
- Set<Class<? extends StreamsDateTimeFormat>> dateTimeFormatClasses = reflections.getSubTypesOf(StreamsDateTimeFormat.class);
-
- List<String> dateTimeFormats = new ArrayList<>();
- for (Class dateTimeFormatClass : dateTimeFormatClasses) {
- try {
- dateTimeFormats.add(((StreamsDateTimeFormat) (dateTimeFormatClass.newInstance())).getFormat());
- } catch (Exception ex) {
- LOGGER.warn("Exception getting format from " + dateTimeFormatClass);
- }
- }
-
- addSerializer(DateTime.class, new StreamsDateTimeSerializer(DateTime.class));
- addDeserializer(DateTime.class, new StreamsDateTimeDeserializer(DateTime.class, dateTimeFormats));
-
- addSerializer(Period.class, new StreamsPeriodSerializer(Period.class));
- addDeserializer(Period.class, new StreamsPeriodDeserializer(Period.class));
- }
-
- public StreamsJacksonModule(List<String> formats) {
- super();
-
- addSerializer(DateTime.class, new StreamsDateTimeSerializer(DateTime.class));
- addDeserializer(DateTime.class, new StreamsDateTimeDeserializer(DateTime.class, formats));
-
- addSerializer(Period.class, new StreamsPeriodSerializer(Period.class));
- addDeserializer(Period.class, new StreamsPeriodDeserializer(Period.class));
- }
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
deleted file mode 100644
index b205a10..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
-import org.joda.time.Period;
-
-import java.io.IOException;
-import java.io.Serializable;
-
-/**
- * StdDeserializer of Period.
- */
-public class StreamsPeriodDeserializer extends StdDeserializer<Period> implements Serializable {
-
- protected StreamsPeriodDeserializer(Class<Period> dateTimeClass) {
- super(dateTimeClass);
- }
-
- public Period deserialize(JsonParser jpar, DeserializationContext context) throws IOException {
- return Period.millis(jpar.getIntValue());
- }
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
deleted file mode 100644
index 3676615..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.jackson;
-
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.ser.std.StdSerializer;
-import org.joda.time.Period;
-
-import java.io.IOException;
-import java.io.Serializable;
-
-/**
- * StdSerializer of Period.
- */
-public class StreamsPeriodSerializer extends StdSerializer<Period> implements Serializable {
-
- protected StreamsPeriodSerializer(Class<Period> dateTimeClass) {
- super(dateTimeClass);
- }
-
- @Override
- public void serialize(Period value, JsonGenerator jgen, SerializerProvider provider) throws IOException {
- jgen.writeString(Integer.toString(value.getMillis()));
- }
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/juneau/JodaDateSwap.java b/streams-pojo/src/main/java/org/apache/streams/juneau/JodaDateSwap.java
deleted file mode 100644
index 2befa33..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/juneau/JodaDateSwap.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.juneau;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.juneau.BeanSession;
-import org.apache.juneau.ClassMeta;
-import org.apache.juneau.parser.ParseException;
-import org.apache.juneau.transform.StringSwap;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
-
-/**
- * Transforms {@link org.joda.time.DateTime} to {@link String Strings}.
- */
-public class JodaDateSwap extends StringSwap<DateTime> {
-
- DateTimeFormatter dateFormatter;
-
- /**
- * Constructor.
- */
- public JodaDateSwap() {
- dateFormatter = ISODateTimeFormat.dateTime();
- }
-
- @Override /* PojoSwap */
- public String swap(BeanSession session, DateTime o) {
- DateTimeFormatter dateFormatter = this.dateFormatter;
- if( StringUtils.isNotBlank(session.getStringProperty("format"))) {
- dateFormatter = DateTimeFormat.forPattern(session.getStringProperty("format"));
- }
- return dateFormatter.print(o);
- }
-
- @Override /* PojoSwap */
- public DateTime unswap(BeanSession session, String f, ClassMeta<?> hint) throws ParseException {
- DateTimeFormatter dateFormatter = this.dateFormatter;
- if( StringUtils.isNotBlank(session.getStringProperty("format"))) {
- dateFormatter = DateTimeFormat.forPattern(session.getStringProperty("format"));
- }
- return dateFormatter.parseDateTime(f);
- }
-
-}
diff --git a/streams-pojo/src/main/java/org/apache/streams/juneau/ListSerializer.java b/streams-pojo/src/main/java/org/apache/streams/juneau/ListSerializer.java
deleted file mode 100644
index 0429087..0000000
--- a/streams-pojo/src/main/java/org/apache/streams/juneau/ListSerializer.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.juneau;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.juneau.PartType;
-import org.apache.juneau.serializer.PartSerializer;
-
-import java.util.List;
-
-/**
- * Serializes {@link java.util.List} as appropriately delimited {@link String Strings}.
- */
-public class ListSerializer implements PartSerializer {
-
- @Override
- public String serialize(PartType type, Object value) {
- List list = (List) value;
- if( list.size() > 0 ) {
- if( type.equals(PartType.QUERY)) {
- return StringUtils.join(list, ",");
- } else if( type.equals(PartType.PATH)) {
- return StringUtils.join(list, "/");
- }
- }
- return null;
- }
-}
diff --git a/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/StreamsJacksonMapperConfiguration.json b/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/StreamsJacksonMapperConfiguration.json
deleted file mode 100644
index 5bfd082..0000000
--- a/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/StreamsJacksonMapperConfiguration.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.pojo.StreamsJacksonMapperConfiguration",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "dateFormats": {
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "enableScala": {
- "type": "boolean",
- "default": false
- }
- }
-}
\ No newline at end of file
diff --git a/streams-pojo/src/site/markdown/index.md b/streams-pojo/src/site/markdown/index.md
deleted file mode 100644
index 3b78d6b..0000000
--- a/streams-pojo/src/site/markdown/index.md
+++ /dev/null
@@ -1,26 +0,0 @@
-streams-pojo
-==============
-
-streams-pojo contains ActivityStream json schema definitions and utility classes.
-
-##Json Schemas
-
-[activity.json](activity.json "activity.json")
-
-[collection.json](collection.json "collection.json")
-
-[media_link.json](media_link.json "media_link.json")
-
-[object.json](object.json "object.json")
-
-[verbs](verbs "verbs/")
-
-[objectTypes](objectTypes "objectTypes/")
-
-##Test Activities
-
-[activities](activities "activities/")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-pojo/src/test/java/org/apache/streams/pojo/test/ActivitySerDeTest.java b/streams-pojo/src/test/java/org/apache/streams/pojo/test/ActivitySerDeTest.java
deleted file mode 100644
index cb0360e..0000000
--- a/streams-pojo/src/test/java/org/apache/streams/pojo/test/ActivitySerDeTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.pojo.test;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.io.IOUtils;
-import org.junit.Test;
-import org.reflections.Reflections;
-import org.reflections.scanners.SubTypesScanner;
-import org.reflections.util.ClasspathHelper;
-import org.reflections.util.ConfigurationBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Test that Activity beans are compatible with the example activities in the spec.
- */
-public class ActivitySerDeTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ActivitySerDeTest.class);
-
- private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
-
- /**
- * Tests that all example activities can be loaded into Activity beans.
- * @throws Exception Exception
- */
- @Test
- public void testActivitySerDe() throws Exception {
-
- InputStream testActivityFolderStream = ActivitySerDeTest.class.getClassLoader()
- .getResourceAsStream("activitystreams-testdocs/activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- for ( String file : files) {
- LOGGER.info("File: " + file );
- LOGGER.info("Serializing: activities/" + file );
- InputStream testActivityFileStream = ActivitySerDeTest.class.getClassLoader()
- .getResourceAsStream("activities/" + file);
- Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
- activity.setGenerator(null);
- activity.setLinks(new LinkedList<>());
- String activityString = MAPPER.writeValueAsString(activity);
- LOGGER.info("Deserialized: " + activityString );
- assert ( !activityString.contains("null") );
- assert ( !activityString.contains("[]") );
- }
- }
-
- /**
- * Tests that defined activity verbs have an example which can be loaded into
- * Activity beans and into verb-specific beans.
- * @throws Exception Exception
- */
- @Test
- public void testVerbSerDe() throws Exception {
-
- Reflections reflections = new Reflections(new ConfigurationBuilder()
- .setUrls(ClasspathHelper.forPackage("org.apache.streams.pojo.json"))
- .setScanners(new SubTypesScanner()));
- Set<Class<? extends Activity>> verbs = reflections.getSubTypesOf(Activity.class);
-
- for ( Class verbClass : verbs) {
- LOGGER.info("Verb: " + verbClass.getSimpleName() );
- Activity activity = (Activity) verbClass.newInstance();
- String verbName = activity.getVerb();
- String testfile = verbName.toLowerCase() + ".json";
- LOGGER.info("Serializing: activities/" + testfile );
- assert (ActivitySerDeTest.class.getClassLoader().getResource("activities/" + testfile) != null);
- InputStream testActivityFileStream = ActivitySerDeTest.class.getClassLoader()
- .getResourceAsStream("activities/" + testfile);
- assert (testActivityFileStream != null);
- activity = MAPPER.convertValue(MAPPER.readValue(testActivityFileStream, verbClass), Activity.class);
- String activityString = MAPPER.writeValueAsString(activity);
- LOGGER.info("Deserialized: " + activityString );
- assert ( !activityString.contains("null") );
- assert ( !activityString.contains("[]") );
- }
- }
-}
diff --git a/streams-pojo/src/test/java/org/apache/streams/pojo/test/CustomDateTimeFormatTest.java b/streams-pojo/src/test/java/org/apache/streams/pojo/test/CustomDateTimeFormatTest.java
deleted file mode 100644
index 8fa927d..0000000
--- a/streams-pojo/src/test/java/org/apache/streams/pojo/test/CustomDateTimeFormatTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.pojo.test;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Assert;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Testing {@link org.apache.streams.jackson.StreamsJacksonMapper} ability to bind
- * custom DateTime formats.
- */
-public class CustomDateTimeFormatTest {
-
- @Test
- public void testCustomDateTimeFormatExplicit() {
- String format = "EEE MMM dd HH:mm:ss Z yyyy";
- String input = "Tue Jan 17 21:21:46 Z 2012";
- Long outputMillis = 1326835306000L;
- ObjectMapper mapper = StreamsJacksonMapper.getInstance(format);
- try {
- String json = "{\"published\":\"" + input + "\"}";
- Activity activity = mapper.readValue(json, Activity.class);
-
- //Writes out value as a String including quotes
- Long result = activity.getPublished().getMillis();
-
- assertEquals(result, outputMillis);
- } catch (Exception ex) {
- ex.printStackTrace();
- Assert.fail();
- }
- }
-
- @Test
- public void testCustomDateTimeFormatReflection() {
- String input = "Tue Jan 17 21:21:46 Z 2012";
- Long outputMillis = 1326835306000L;
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- try {
- String json = "{\"published\":\"" + input + "\"}";
- Activity activity = mapper.readValue(json, Activity.class);
-
- //Writes out value as a String including quotes
- Long result = activity.getPublished().getMillis();
-
- assertEquals(result, outputMillis);
- } catch (Exception ex) {
- ex.printStackTrace();
- Assert.fail();
- }
- }
-
-
-}
diff --git a/streams-pojo/src/test/java/org/apache/streams/pojo/test/DateTimeSerDeTest.java b/streams-pojo/src/test/java/org/apache/streams/pojo/test/DateTimeSerDeTest.java
deleted file mode 100644
index 9a7a5ac..0000000
--- a/streams-pojo/src/test/java/org/apache/streams/pojo/test/DateTimeSerDeTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.pojo.test;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.joda.time.DateTime;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Testing {@link org.apache.streams.jackson.StreamsJacksonMapper} ability to read and write
- * RFC3339 format.
- */
-public class DateTimeSerDeTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(DateTimeSerDeTest.class);
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- @Ignore
- @Test
- // this really needs to be able to pass...
- public void testActivityStringSer() {
- String input = "2012-01-17T21:21:46.000Z";
- try {
- DateTime output = mapper.readValue(input, DateTime.class);
- } catch (Exception ex) {
- ex.printStackTrace();
- Assert.fail();
- }
- }
-
- @Test
- public void testMillisDeser() {
- Long input = 1326856906000L;
- try {
- DateTime output = mapper.readValue(input.toString(), DateTime.class);
- } catch (Exception ex) {
- ex.printStackTrace();
- Assert.fail();
- }
- }
-
- @Test
- public void testActivityStringDeser() {
- String output = "2012-01-17T21:21:46.000Z";
- long inputMillis = 1326835306000L;
- DateTime input;
- try {
- input = new DateTime(inputMillis);
- //Writes out value as a String including quotes
- String result = mapper.writeValueAsString(input);
- assertEquals(result.replace("\"", ""), output);
- } catch (Exception ex) {
- ex.printStackTrace();
- Assert.fail();
- }
- }
-
-}
diff --git a/streams-pojo/src/test/java/org/apache/streams/pojo/test/RFC3339UtilsTest.java b/streams-pojo/src/test/java/org/apache/streams/pojo/test/RFC3339UtilsTest.java
deleted file mode 100644
index bce756a..0000000
--- a/streams-pojo/src/test/java/org/apache/streams/pojo/test/RFC3339UtilsTest.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-
-package org.apache.streams.pojo.test;
-
-import org.apache.streams.data.util.RFC3339Utils;
-
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.junit.Test;
-
-import java.util.TimeZone;
-
-import static org.apache.streams.data.util.RFC3339Utils.format;
-import static org.apache.streams.data.util.RFC3339Utils.parseUTC;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
-
-/**
- * Testing {@link org.apache.streams.data.util.RFC3339Utils}
- */
-public class RFC3339UtilsTest {
-
- @Test
- public void validUTC() {
- DateTime parsed = parseUTC("2014-12-25T12:00:00Z");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(12)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- }
-
- @Test
- public void validUTCSubSecond() {
- DateTime parsed = parseUTC("2014-12-25T12:00:00.7Z");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(12)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- assertThat(parsed.millisOfSecond().get(), is(equalTo(700)));
- }
-
- @Test
- public void validUTCSubSecondMultiDigit() {
- DateTime parsed = parseUTC("2014-12-25T12:00:00.7343Z");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(12)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- assertThat(parsed.millisOfSecond().get(), is(equalTo(734)));
- }
-
- @Test
- public void validEST() {
- DateTime parsed = parseUTC("2014-12-25T12:00:00-05:00");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(17)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- }
-
- @Test
- public void validESTSubSecond() {
- DateTime parsed = parseUTC("2014-12-25T12:00:00.7-05:00");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(17)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- assertThat(parsed.millisOfSecond().get(), is(equalTo(700)));
- }
-
- @Test
- public void validESTSubSecondMultiDigit() {
- DateTime parsed = parseUTC("2014-12-25T12:00:00.7343-05:00");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(17)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- assertThat(parsed.millisOfSecond().get(), is(equalTo(734)));
- }
-
- @Test
- public void validESTNoSeparator() {
- DateTime parsed = parseUTC("2014-12-25T12:00:00-0500");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(17)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- }
-
- @Test
- public void validESTSubSecondNoSeparator() {
- DateTime parsed = parseUTC("2014-12-25T12:00:00.7-0500");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(17)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- assertThat(parsed.millisOfSecond().get(), is(equalTo(700)));
- }
-
- @Test
- public void validESTSubSecondMultiDigitNoSeparator() {
- DateTime parsed = parseUTC("2014-12-25T12:00:00.7343-0500");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(17)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- assertThat(parsed.millisOfSecond().get(), is(equalTo(734)));
- }
-
- @Test
- public void validCET() {
- DateTime parsed = parseUTC("2014-12-25T12:00:00+01:00");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(11)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- }
-
- @Test
- public void validCETSubSecond() {
- DateTime parsed = parseUTC("2014-12-25T12:00:00.7+01:00");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(11)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- assertThat(parsed.millisOfSecond().get(), is(equalTo(700)));
- }
-
- @Test
- public void validCETSubSecondMultidigit() {
- DateTime parsed = parseUTC("2014-12-25T12:00:00.7343+01:00");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(11)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- assertThat(parsed.millisOfSecond().get(), is(equalTo(734)));
- }
-
- @Test
- public void validLong() {
- DateTime parsed = parseUTC("1419505200734");
- assertThat(parsed.minuteOfHour().get(), is(equalTo(0)));
- assertThat(parsed.hourOfDay().get(), is(equalTo(11)));
- assertThat(parsed.dayOfMonth().get(), is(equalTo(25)));
- assertThat(parsed.monthOfYear().get(), is(equalTo(12)));
- assertThat(parsed.millisOfSecond().get(), is(equalTo(734)));
- }
-
- @Test
- public void validFormatUTC() {
- DateTime parsed = new DateTime(1419505200734L);
- assertThat(format(parsed), is(equalTo("2014-12-25T11:00:00.734Z")));
- }
-
- @Test
- public void validFormat() {
- TimeZone cet = TimeZone.getTimeZone("CET");
- DateTime parsed = new DateTime(1419505200734L);
- assertThat(format(parsed, cet), is(equalTo("2014-12-25T12:00:00.734+0100")));
- }
-
- @Test
- public void testParseVariousDateFormats() {
- String date = "Thu April 24 04:43:10 -0500 2014";
- DateTime expected = new DateTime(2014, 4, 24, 9, 43, 10, DateTimeZone.forOffsetHours(0));
- testHelper(expected, date);
- date = "2014/04/24 04:43:10";
- expected = new DateTime(2014, 4, 24, 4, 43, 10, DateTimeZone.forOffsetHours(0));
- testHelper(expected, date);
- date = "2014-04-24T04:43:10Z";
- testHelper(expected, date);
- date = "04:43:10 2014/04/24";
- testHelper(expected, date);
- date = "4/24/2014 04:43:10";
- testHelper(expected, date);
- date = "04:43:10 4/24/2014";
- testHelper(expected, date);
- date = "04:43:10 2014-04-24";
- testHelper(expected, date);
- date = "4-24-2014 04:43:10";
- testHelper(expected, date);
- date = "04:43:10 4-24-2014";
- testHelper(expected, date);
- expected = new DateTime(2014, 4, 24, 0, 0, 0, DateTimeZone.forOffsetHours(0));
- date = "24-4-2014";
- testHelper(expected, date);
- date = "2014-4-24";
- testHelper(expected, date);
- date = "2014/4/24";
- testHelper(expected, date);
- date = "2014/4/24 fesdfs";
- try {
- RFC3339Utils.parseToUTC(date);
- fail("Should not have been able to parse : " + date);
- } catch (Exception ex) {
- //
- }
- }
-
- private void testHelper(DateTime expected, String dateString) {
- DateTime parsedDate = RFC3339Utils.parseToUTC(dateString);
- assertEquals("Failed to parse : " + dateString, expected, parsedDate);
- String rfc3339String = RFC3339Utils.format(dateString);
- String parsedRfc3339String = RFC3339Utils.format(parsedDate);
- assertEquals("Parsed String should be equal.", parsedRfc3339String, rfc3339String);
- DateTime convertedBack = RFC3339Utils.parseToUTC(parsedRfc3339String);
- assertEquals(expected, convertedBack);
- }
-}
diff --git a/streams-runtimes/README.md b/streams-runtimes/README.md
deleted file mode 100644
index ce532d2..0000000
--- a/streams-runtimes/README.md
+++ /dev/null
@@ -1,11 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-streams-runtimes
-==============
-
-These modules contain builders and helpers that support instantiation,
-setup, execution, and cleanup of streams in a specific runtime context.
-
-Please help improve these packages, or create new ones!
diff --git a/streams-runtimes/pom.xml b/streams-runtimes/pom.xml
deleted file mode 100644
index a5ff86c..0000000
--- a/streams-runtimes/pom.xml
+++ /dev/null
@@ -1,40 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>apache-streams</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-runtimes</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Apache Streams Runtimes</description>
-
- <packaging>pom</packaging>
-
- <modules>
- <module>streams-runtime-dropwizard</module>
- <module>streams-runtime-local</module>
- </modules>
-</project>
diff --git a/streams-runtimes/streams-runtime-dropwizard/README.md b/streams-runtimes/streams-runtime-dropwizard/README.md
deleted file mode 100644
index e68bf16..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-runtime-dropwizard
-=============================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-runtimes/streams-runtime-dropwizard/pom.xml b/streams-runtimes/streams-runtime-dropwizard/pom.xml
deleted file mode 100644
index 79d30ab..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/pom.xml
+++ /dev/null
@@ -1,262 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-runtimes</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-runtime-dropwizard</artifactId>
-
- <properties>
- <dropwizard.version>0.7.1</dropwizard.version>
- </properties>
-
- <dependencies>
-
- <dependency>
- <groupId>io.dropwizard</groupId>
- <artifactId>dropwizard-core</artifactId>
- <version>${dropwizard.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-afterburner</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>io.dropwizard</groupId>
- <artifactId>dropwizard-configuration</artifactId>
- <version>${dropwizard.version}</version>
- </dependency>
-
- <dependency>
- <groupId>io.dropwizard</groupId>
- <artifactId>dropwizard-validation</artifactId>
- <version>${dropwizard.version}</version>
- </dependency>
-
- <dependency>
- <groupId>io.dropwizard</groupId>
- <artifactId>dropwizard-metrics</artifactId>
- <version>${dropwizard.version}</version>
- </dependency>
-
- <dependency>
- <groupId>io.dropwizard</groupId>
- <artifactId>dropwizard-testing</artifactId>
- <version>${dropwizard.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-afterburner</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-guava</artifactId>
- <version>${jackson.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.hubspot.dropwizard</groupId>
- <artifactId>dropwizard-guice</artifactId>
- <version>0.7.1</version>
- </dependency>
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- <version>3.0</version>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-runtime-local</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-core</artifactId>
- </dependency>
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>log4j-over-slf4j</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-all</artifactId>
- <version>1.3</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>1.9.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.dropwizard.pojo</targetPackage>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>io.dropwizard</groupId>
- <artifactId>dropwizard-core</artifactId>
- <version>${dropwizard.version}</version>
- </dependency>
- <dependency>
- <groupId>io.dropwizard</groupId>
- <artifactId>dropwizard-configuration</artifactId>
- <version>${dropwizard.version}</version>
- </dependency>
- <dependency>
- <groupId>io.dropwizard</groupId>
- <artifactId>dropwizard-validation</artifactId>
- <version>${dropwizard.version}</version>
- </dependency>
- <dependency>
- <groupId>io.dropwizard</groupId>
- <artifactId>dropwizard-metrics</artifactId>
- <version>${dropwizard.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- </plugin>
- <!--<plugin>-->
- <!--<groupId>org.apache.maven.plugins</groupId>-->
- <!--<artifactId>maven-resources-plugin</artifactId>-->
- <!--<resources>test.</resources>-->
- <!--</plugin>-->
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/GenericWebhookResource.java b/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/GenericWebhookResource.java
deleted file mode 100644
index 56fb465..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/GenericWebhookResource.java
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.dropwizard;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.ComponentUtils;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Splitter;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.math.BigInteger;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.regex.Pattern;
-import javax.annotation.Resource;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-/**
- * GenericWebhookResource provides basic webhook connectivity.
- *
- * <p/>
- * Add processors / persistWriters that read from "GenericWebhookResource" to
- * consume data posted to streams.
- */
-@Resource
-@Path("/streams/webhooks")
-@Produces(MediaType.APPLICATION_JSON)
-@Consumes(MediaType.APPLICATION_JSON)
-public class GenericWebhookResource implements StreamsProvider {
-
- public static final String STREAMS_ID = "GenericWebhookResource";
-
- public GenericWebhookResource() {
- }
-
- private static final Logger log = LoggerFactory
- .getLogger(GenericWebhookResource.class);
-
- private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- protected Queue<StreamsDatum> providerQueue = new ConcurrentLinkedQueue<>();
-
- protected final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- private static Pattern newLinePattern = Pattern.compile("(\\r\\n?|\\n)", Pattern.MULTILINE);
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- /**
- * push a String json datum into a stream.
- * @param headers HttpHeaders
- * @param body String json
- * @return Response
- */
- @POST
- @Path("json")
- public Response json(@Context HttpHeaders headers,
- String body) {
-
- ObjectNode response = mapper.createObjectNode();
- int responseCode = Response.Status.BAD_REQUEST.getStatusCode();
-
- try {
- ObjectNode item = mapper.readValue(body, ObjectNode.class);
-
- StreamsDatum datum = new StreamsDatum(body);
-
- lock.writeLock().lock();
- ComponentUtils.offerUntilSuccess(datum, providerQueue);
- lock.writeLock().unlock();
-
- Boolean success = true;
-
- response.put("success", success);
-
- responseCode = Response.Status.OK.getStatusCode();
-
- } catch (Exception ex) {
- log.warn(ex.toString(), ex);
-
- Boolean success = false;
-
- response.put("success", success);
- responseCode = Response.Status.BAD_REQUEST.getStatusCode();
-
- } finally {
- return Response.status(responseCode).entity(response).build();
- }
- }
-
- /**
- * push multiple String json datums into a stream.
- * @param headers HttpHeaders
- * @param body String json
- * @return Response
- */
- @POST
- @Path("json_new_line")
- public Response json_new_line(@Context HttpHeaders headers,
- String body) {
-
- ObjectNode response = mapper.createObjectNode();
- int responseCode = Response.Status.BAD_REQUEST.getStatusCode();
-
- if (body.equalsIgnoreCase("{}")) {
-
- Boolean success = true;
-
- response.put("success", success);
- responseCode = Response.Status.OK.getStatusCode();
- return Response.status(responseCode).entity(response).build();
- }
-
- try {
-
- for (String line : Splitter.on(newLinePattern).split(body)) {
- ObjectNode item = mapper.readValue(line, ObjectNode.class);
-
- StreamsDatum datum = new StreamsDatum(item);
-
- lock.writeLock().lock();
- ComponentUtils.offerUntilSuccess(datum, providerQueue);
- lock.writeLock().unlock();
-
- }
-
- Boolean success = true;
-
- response.put("success", success);
- responseCode = Response.Status.OK.getStatusCode();
-
- } catch (Exception ex) {
- log.warn(ex.toString(), ex);
-
- Boolean success = false;
-
- response.put("success", success);
- responseCode = Response.Status.BAD_REQUEST.getStatusCode();
-
- } finally {
- return Response.status(responseCode).entity(response).build();
-
- }
-
- }
-
- /**
- * push multiple ObjectNode json datums into a stream.
- * @param headers HttpHeaders
- * @param body String json
- * @return Response
- */
- @POST
- @Path("json_meta")
- public Response json_meta(@Context HttpHeaders headers,
- String body) {
-
- ObjectNode response = mapper.createObjectNode();
- int responseCode = Response.Status.BAD_REQUEST.getStatusCode();
-
- if (body.equalsIgnoreCase("{}")) {
-
- Boolean success = true;
-
- response.put("success", success);
- responseCode = Response.Status.OK.getStatusCode();
-
- return Response.status(responseCode).entity(response).build();
- }
-
- try {
-
- GenericWebhookData objectWrapper = mapper.readValue(body, GenericWebhookData.class);
-
- for ( ObjectNode item : objectWrapper.getData()) {
-
- StreamsDatum datum = new StreamsDatum(item);
-
- lock.writeLock().lock();
- ComponentUtils.offerUntilSuccess(datum, providerQueue);
- lock.writeLock().unlock();
- }
-
- Boolean success = true;
-
- response.put("success", success);
- responseCode = Response.Status.OK.getStatusCode();
-
- } catch (Exception ex) {
- log.warn(ex.toString(), ex);
-
- Boolean success = false;
-
- response.put("success", success);
- responseCode = Response.Status.BAD_REQUEST.getStatusCode();
- } finally {
- return Response.status(responseCode).entity(response).build();
- }
-
- }
-
- @Override
- public void startStream() {
- }
-
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- lock.writeLock().lock();
- current = new StreamsResultSet(new ConcurrentLinkedQueue<>(providerQueue));
- providerQueue.clear();
- lock.writeLock().unlock();
-
- return current;
-
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return null;
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
- }
-
- @Override
- public boolean isRunning() {
- return true;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
-
- }
-
-}
\ No newline at end of file
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java b/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java
deleted file mode 100644
index f13a41f..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.dropwizard;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.core.StreamBuilder;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.local.LocalRuntimeConfiguration;
-import org.apache.streams.local.builders.LocalStreamBuilder;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import java.util.Map;
-
-/**
- * StreamDropwizardBuilder is currently a light wrapper around LocalStreamBuilder.
- *
- * <p/>
- * It's a seperate class because they will almost certainly deviate going forward
- */
-public class StreamDropwizardBuilder extends LocalStreamBuilder implements StreamBuilder {
-
- public StreamDropwizardBuilder() {
- super();
- }
-
- public StreamDropwizardBuilder(StreamsConfiguration streamConfig) {
- super(new ObjectMapper().convertValue(streamConfig, LocalRuntimeConfiguration.class));
- }
-
- public StreamDropwizardBuilder(Map<String, Object> streamConfig) {
- super(streamConfig);
- }
-
- public StreamDropwizardBuilder(int maxQueueCapacity) {
- super(maxQueueCapacity);
- }
-
- public StreamDropwizardBuilder(int maxQueueCapacity, Map<String, Object> streamConfig) {
- super(maxQueueCapacity, streamConfig);
- }
-
- @Override
- public StreamBuilder newPerpetualStream(String streamId, StreamsProvider provider) {
- return super.newPerpetualStream(streamId, provider);
- }
-
-}
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java b/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java
deleted file mode 100644
index 38d0f7b..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.dropwizard;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.core.StreamBuilder;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.codahale.metrics.MetricRegistry;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.datatype.guava.GuavaModule;
-import com.fasterxml.jackson.module.afterburner.AfterburnerModule;
-import com.hubspot.dropwizard.guice.GuiceBundle;
-import io.dropwizard.Application;
-import io.dropwizard.jackson.GuavaExtrasModule;
-import io.dropwizard.metrics.MetricsFactory;
-import io.dropwizard.setup.Bootstrap;
-import io.dropwizard.setup.Environment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
-
-/**
- * Entry point to a dropwizard streams application
- *
- * <p/>
- * It will start up a stream in the local runtime, as well as bind any
- * StreamsProvider on the classpath with a @Resource annotation.
- */
-public class StreamsApplication extends Application<StreamsDropwizardConfiguration> {
-
- private static final Logger LOGGER = LoggerFactory
- .getLogger(StreamsApplication.class);
-
- protected static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- protected StreamBuilder builder;
-
- private static StreamsConfiguration streamsConfiguration;
-
- // ConcurrentHashSet is preferable, but it's only in guava 15+
- // spark 1.5.0 uses guava 14 so for the moment this is the workaround
- // Set<StreamsProvider> resourceProviders = Sets.newConcurrentHashSet();
- private Set<StreamsProvider> resourceProviders = Collections.newSetFromMap(new ConcurrentHashMap<StreamsProvider, Boolean>());
-
- private Executor executor = Executors.newSingleThreadExecutor();
-
- static {
- mapper.registerModule(new AfterburnerModule());
- mapper.registerModule(new GuavaModule());
- mapper.registerModule(new GuavaExtrasModule());
- }
-
- @Override
- public void initialize(Bootstrap<StreamsDropwizardConfiguration> bootstrap) {
-
- LOGGER.info(getClass().getPackage().getName());
-
- GuiceBundle<StreamsDropwizardConfiguration> guiceBundle =
- GuiceBundle.<StreamsDropwizardConfiguration>newBuilder()
- .addModule(new StreamsDropwizardModule())
- .setConfigClass(StreamsDropwizardConfiguration.class)
- // override and add more packages to pick up custom Resources
- .enableAutoConfig(getClass().getPackage().getName())
- .build();
- bootstrap.addBundle(guiceBundle);
-
- }
-
- @Override
- public void run(StreamsDropwizardConfiguration streamsDropwizardConfiguration, Environment environment) throws Exception {
-
- executor = Executors.newSingleThreadExecutor();
-
- for ( Class<?> resourceProviderClass : environment.jersey().getResourceConfig().getRootResourceClasses() ) {
- StreamsProvider provider = (StreamsProvider)resourceProviderClass.newInstance();
- if ( StreamsProvider.class.isInstance(provider)) {
- resourceProviders.add(provider);
- }
- }
-
- MetricRegistry metrics = new MetricRegistry();
- MetricsFactory mfac = streamsDropwizardConfiguration.getMetricsFactory();
- mfac.configure(environment.lifecycle(), metrics);
-
- streamsConfiguration = mapper.convertValue(streamsDropwizardConfiguration, StreamsConfiguration.class);
-
- builder = setup(streamsConfiguration, resourceProviders);
-
- executor.execute(new StreamsDropwizardRunner(builder, streamsConfiguration));
-
- // wait for streams to start up
- Thread.sleep(10000);
-
- for (StreamsProvider resource : resourceProviders) {
- environment.jersey().register(resource);
- LOGGER.info("Added resource class: {}", resource);
- }
-
- }
-
- /**
- * setup StreamBuilder.
- * @param streamsConfiguration StreamsConfiguration
- * @param resourceProviders Set of StreamsProvider
- * @return StreamBuilder
- */
- public StreamBuilder setup(StreamsConfiguration streamsConfiguration, Set<StreamsProvider> resourceProviders) {
-
- StreamBuilder builder = new StreamDropwizardBuilder(streamsConfiguration);
-
- List<String> providers = new ArrayList<>();
- for ( StreamsProvider provider: resourceProviders) {
- String providerId = provider.getClass().getSimpleName();
- builder.newPerpetualStream(providerId, provider);
- providers.add(providerId);
- }
-
- return builder;
- }
-
- private class StreamsDropwizardRunner implements Runnable {
-
- private StreamsConfiguration streamsConfiguration;
-
- private StreamBuilder builder;
-
- protected StreamsDropwizardRunner(StreamBuilder builder, StreamsConfiguration streamsConfiguration) {
- this.streamsConfiguration = streamsConfiguration;
- this.builder = builder;
- }
-
- @Override
- public void run() {
-
- builder.start();
-
- }
- }
-
- /**
- * Run from console:
- *
- * <p/>
- * java -jar uber.jar server ./configuration.yml
- *
- * @param args ["server", configuration.yml]
- * @throws Exception Exception
- */
- public static void main(String[] args) throws Exception {
-
- new StreamsApplication().run(args);
-
- }
-
-}
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsDropwizardModule.java b/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsDropwizardModule.java
deleted file mode 100644
index 9514caf..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsDropwizardModule.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.dropwizard;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Provides;
-import com.google.inject.Singleton;
-
-/**
- * This class exists because dropwizard-guice requires at least
- * one module to run.
- *
- * <p/>
- * Do not expect @Inject StreamsConfiguration to work at the moment.
- */
-public class StreamsDropwizardModule extends AbstractModule {
-
- @Override
- protected void configure() {
- requestStaticInjection(StreamsConfiguration.class);
- }
-
- @Provides
- @Singleton
- public StreamsConfiguration providesStreamsConfiguration() {
- return StreamsConfigurator.detectConfiguration();
- }
-
-}
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/main/jsonschema/org/apache/streams/dropwizard/GenericWebhookData.json b/streams-runtimes/streams-runtime-dropwizard/src/main/jsonschema/org/apache/streams/dropwizard/GenericWebhookData.json
deleted file mode 100644
index adcafe2..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/src/main/jsonschema/org/apache/streams/dropwizard/GenericWebhookData.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "$schema": "http://json-schema.org/draft-03/schema",
- "javaType": "org.apache.streams.dropwizard.GenericWebhookData",
- "properties": {
- "id": {
- "type": "string"
- },
- "hash": {
- "type": "string"
- },
- "hash_type": {
- "type": "string"
- },
- "count": {
- "type": "long"
- },
- "delivered_at": {
- "type": "string",
- "format": "date-time"
- },
- "data": {
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "com.fasterxml.jackson.databind.node.ObjectNode"
- }
- }
- }
-}
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/main/jsonschema/org/apache/streams/dropwizard/StreamsDropwizardConfiguration.json b/streams-runtimes/streams-runtime-dropwizard/src/main/jsonschema/org/apache/streams/dropwizard/StreamsDropwizardConfiguration.json
deleted file mode 100644
index 59628ba..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/src/main/jsonschema/org/apache/streams/dropwizard/StreamsDropwizardConfiguration.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.dropwizard.StreamsDropwizardConfiguration",
- "extends": {
- "javaType": "io.dropwizard.Configuration",
- "type": "object"
- }
-}
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/site/markdown/index.md b/streams-runtimes/streams-runtime-dropwizard/src/site/markdown/index.md
deleted file mode 100644
index 1c8e13e..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/src/site/markdown/index.md
+++ /dev/null
@@ -1,16 +0,0 @@
-streams-runtime-dropwizard
-==============
-
-streams-runtime-dropwizard can embed streams and stream components within the dropwizard framework.
-
-## Data Types
-
-[GenericWebhookData.json](GenericWebhookData.json "GenericWebhookData.json")
-
-## Configuration
-
-[StreamsDropwizardConfiguration.json](StreamsDropwizardConfiguration.json "StreamsDropwizardConfiguration.json")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/test/java/org/apache/streams/dropwizard/test/GenericWebhookResourceTest.java b/streams-runtimes/streams-runtime-dropwizard/src/test/java/org/apache/streams/dropwizard/test/GenericWebhookResourceTest.java
deleted file mode 100644
index 68c2ec7..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/src/test/java/org/apache/streams/dropwizard/test/GenericWebhookResourceTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.dropwizard.test;
-
-import org.apache.streams.dropwizard.GenericWebhookData;
-import org.apache.streams.dropwizard.GenericWebhookResource;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import io.dropwizard.testing.junit.ResourceTestRule;
-import org.joda.time.DateTime;
-import org.junit.Assert;
-import org.junit.ClassRule;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Tests {@link: org.apache.streams.dropwizard.GenericWebhookResource}
- */
-public class GenericWebhookResourceTest {
-
- private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- private static final GenericWebhookResource genericWebhookResource = new GenericWebhookResource();
-
- @ClassRule
- public static final ResourceTestRule resources = ResourceTestRule.builder()
- .addResource(genericWebhookResource)
- .build();
-
- @Test
- public void testPostJson() {
- Assert.assertEquals(400, genericWebhookResource.json(null, "{").getStatus());
- Assert.assertEquals(400, genericWebhookResource.json(null, "}").getStatus());
- Assert.assertEquals(400, genericWebhookResource.json(null, "srg").getStatus());
- Assert.assertEquals(400, genericWebhookResource.json(null, "123").getStatus());
- Assert.assertEquals(200, genericWebhookResource.json(null, "{}").getStatus());
- Assert.assertEquals(200, genericWebhookResource.json(null, "{\"valid\":\"true\"}").getStatus());
- }
-
- @Test
- public void testPostJsonNewLine() {
- Assert.assertEquals(200, genericWebhookResource.json_new_line(null, "{}").getStatus());
- Assert.assertEquals(400, genericWebhookResource.json_new_line(null, "notvalid").getStatus());
- Assert.assertEquals(200, genericWebhookResource.json_new_line(null, "{\"valid\":\"true\"}").getStatus());
- Assert.assertEquals(200, genericWebhookResource.json_new_line(null, "{\"valid\":\"true\"}\n{\"valid\":\"true\"}\r{\"valid\":\"true\"}").getStatus());
- }
-
- @Test
- public void testPostJsonMeta() throws JsonProcessingException {
- Assert.assertEquals(200, genericWebhookResource.json_meta(null, "{}").getStatus());
- Assert.assertEquals(400, genericWebhookResource.json_meta(null, "notvalid").getStatus());
- GenericWebhookData testPostJsonMeta = new GenericWebhookData()
- .withHash("test")
- .withDeliveredAt(DateTime.now())
- .withCount(1)
- .withHashType("type")
- .withId("test");
- List<ObjectNode> testPostJsonData = new ArrayList<>();
- testPostJsonData.add(mapper.createObjectNode().put("valid", "true"));
- testPostJsonMeta.setData(testPostJsonData);
- String testPostJsonEntity = mapper.writeValueAsString(testPostJsonMeta);
- Assert.assertEquals(200, genericWebhookResource.json_meta(null, testPostJsonEntity).getStatus());
-
- }
-
-}
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/test/java/org/apache/streams/dropwizard/test/StreamsApplicationIT.java b/streams-runtimes/streams-runtime-dropwizard/src/test/java/org/apache/streams/dropwizard/test/StreamsApplicationIT.java
deleted file mode 100644
index 743df02..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/src/test/java/org/apache/streams/dropwizard/test/StreamsApplicationIT.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.dropwizard.test;
-
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests {@link: org.apache.streams.dropwizard.StreamsApplication}
- */
-public class StreamsApplicationIT {
-
- @BeforeClass
- public void setupTest() throws Exception {
- String[] testArgs = Stream.of("server", "src/test/resources/configuration.yml").collect(Collectors.toList()).toArray(new String[2]);
- TestStreamsApplication.main(testArgs);
- }
-
- @Test
- public void testApplicationStarted() throws Exception {
-
- final URL url = new URL("http://localhost:8003/admin/ping");
- final String response = new BufferedReader(new InputStreamReader(url.openStream())).readLine();
- Assert.assertEquals("pong", response);
- }
-}
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/test/java/org/apache/streams/dropwizard/test/TestStreamsApplication.java b/streams-runtimes/streams-runtime-dropwizard/src/test/java/org/apache/streams/dropwizard/test/TestStreamsApplication.java
deleted file mode 100644
index 9c10b93..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/src/test/java/org/apache/streams/dropwizard/test/TestStreamsApplication.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.dropwizard.test;
-
-import org.apache.streams.dropwizard.StreamsApplication;
-
-/**
- * This class exists to support {@link: org.apache.streams.dropwizard.test.StreamsApplicationIT}
- */
-public class TestStreamsApplication extends StreamsApplication {
-}
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/test/resources/configuration.yml b/streams-runtimes/streams-runtime-dropwizard/src/test/resources/configuration.yml
deleted file mode 100644
index 6517565..0000000
--- a/streams-runtimes/streams-runtime-dropwizard/src/test/resources/configuration.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-template: Hello, %s!
-defaultName: streams
-
-server:
- type: simple
- applicationContextPath: /
- adminContextPath: /admin
- connector:
- type: http
- port: 8003
-
-logging:
- level: DEBUG
- appenders:
- - type: console
- threshold: ALL
- target: stdout
diff --git a/streams-runtimes/streams-runtime-local/README.md b/streams-runtimes/streams-runtime-local/README.md
deleted file mode 100644
index d4425ae..0000000
--- a/streams-runtimes/streams-runtime-local/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-runtime-local
-========================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-runtimes/streams-runtime-local/pom.xml b/streams-runtimes/streams-runtime-local/pom.xml
deleted file mode 100644
index de163c2..0000000
--- a/streams-runtimes/streams-runtime-local/pom.xml
+++ /dev/null
@@ -1,176 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-runtimes</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-runtime-local</artifactId>
- <name>streams-runtime-local</name>
-
- <description>Streams Local Runtime</description>
-
- <dependencies>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-core</artifactId>
- </dependency>
- <dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>log4j-over-slf4j</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-monitoring</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-all</artifactId>
- <version>1.3</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>1.9.5</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.local</targetPackage>
- </configuration>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-config</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-monitoring</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes combine.children="append">
- <exclude>src/test/resources/TestFile.txt</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-</project>
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/InvalidStreamException.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/InvalidStreamException.java
deleted file mode 100644
index c44a8e4..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/InvalidStreamException.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.builders;
-
-/**
- * Exception that indicates a malformed data stream in some way.
- */
-public class InvalidStreamException extends RuntimeException {
-
- public InvalidStreamException() {
- super();
- }
-
- public InvalidStreamException(String string) {
- super(string);
- }
-
- public InvalidStreamException(String string, Throwable throwable) {
- super(string, throwable);
- }
-
- public InvalidStreamException(Throwable throwable) {
- super(throwable);
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
deleted file mode 100644
index 165010f..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.builders;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatusCountable;
-import org.apache.streams.core.StreamBuilder;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.local.LocalRuntimeConfiguration;
-import org.apache.streams.local.counters.StreamsTaskCounter;
-import org.apache.streams.local.executors.ShutdownStreamOnUnhandleThrowableThreadPoolExecutor;
-import org.apache.streams.local.monitoring.MonitoringConfiguration;
-import org.apache.streams.local.queues.ThroughputQueue;
-import org.apache.streams.local.tasks.BaseStreamsTask;
-import org.apache.streams.local.tasks.LocalStreamProcessMonitorThread;
-import org.apache.streams.local.tasks.StatusCounterMonitorThread;
-import org.apache.streams.local.tasks.StreamsProviderTask;
-import org.apache.streams.local.tasks.StreamsTask;
-import org.apache.streams.monitoring.tasks.BroadcastMonitorThread;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-
-import java.math.BigInteger;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-/**
- * {@link org.apache.streams.local.builders.LocalStreamBuilder} implementation to run a data processing stream in a single
- * JVM across many threads. Depending on your data stream, the JVM heap may need to be set to a high value. Default
- * implementation uses unbound {@link java.util.concurrent.ConcurrentLinkedQueue} to connect stream components.
- */
-public class LocalStreamBuilder implements StreamBuilder {
-
- private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(LocalStreamBuilder.class);
- private static final int DEFAULT_QUEUE_SIZE = 500;
-
- public static final String TIMEOUT_KEY = "TIMEOUT";
- public static final String BROADCAST_KEY = "broadcastURI";
- public static final String STREAM_IDENTIFIER_KEY = "streamsID";
- public static final String BROADCAST_INTERVAL_KEY = "monitoring_broadcast_interval_ms";
- public static final String DEFAULT_STREAM_IDENTIFIER = "Unknown_Stream";
- public static final String DEFAULT_STARTED_AT_KEY = "startedAt";
-
- private Map<String, StreamComponent> providers;
- private Map<String, StreamComponent> components;
- private LocalRuntimeConfiguration streamConfig;
- private Map<StreamsTask, Future> futures;
- private ExecutorService executor;
- private ExecutorService monitor;
- private int totalTasks;
- private int monitorTasks;
- private LocalStreamProcessMonitorThread monitorThread;
- private Map<String, List<StreamsTask>> tasks;
- private Thread shutdownHook;
- private BroadcastMonitorThread broadcastMonitor;
- private int maxQueueCapacity;
- private String streamIdentifier = DEFAULT_STREAM_IDENTIFIER;
- private DateTime startedAt = new DateTime();
- private boolean useDeprecatedMonitors;
-
- /**
- * Creates a local stream builder with all configuration resolved by typesafe
- */
- public LocalStreamBuilder() {
- this(new ObjectMapper().convertValue(StreamsConfigurator.detectConfiguration(), LocalRuntimeConfiguration.class));
- }
-
- /**
- * Creates a local stream builder with a config object and default maximum internal queue size of 500
- * @param streamConfig
- * @deprecated use LocalRuntimeConfiguration constructor instead
- */
- @Deprecated
- public LocalStreamBuilder(Map<String, Object> streamConfig) {
- this(DEFAULT_QUEUE_SIZE, streamConfig);
- }
-
- /**
- * Creates a local stream builder with no config object. If maxQueueCapacity is less than 1 the queue is
- * unbounded.
- * @param maxQueueCapacity
- *
- * @deprecated use LocalRuntimeConfiguration constructor instead
- */
- @Deprecated
- public LocalStreamBuilder(int maxQueueCapacity) {
- this(maxQueueCapacity, null);
- }
-
- /**
- * Creates a local stream builder with a config object. If maxQueueCapacity is less than 1 the queue is
- * unbounded.
- *
- * @param maxQueueCapacity
- * @param streamConfig
- *
- * @deprecated use LocalRuntimeConfiguration constructor instead
- */
- @Deprecated
- public LocalStreamBuilder(int maxQueueCapacity, Map<String, Object> streamConfig) {
- this(new ObjectMapper().convertValue(StreamsConfigurator.detectConfiguration(), LocalRuntimeConfiguration.class));
- this.streamConfig.setQueueSize((long) maxQueueCapacity);
- if( streamConfig != null && streamConfig.get(LocalStreamBuilder.TIMEOUT_KEY) != null )
- this.streamConfig.setProviderTimeoutMs(new Long((Integer) (streamConfig.get(LocalStreamBuilder.TIMEOUT_KEY))));
- if( streamConfig != null && streamConfig.get(LocalStreamBuilder.STREAM_IDENTIFIER_KEY) != null )
- this.streamConfig.setIdentifier((String)streamConfig.get(LocalStreamBuilder.STREAM_IDENTIFIER_KEY));
- if( streamConfig != null && streamConfig.get(LocalStreamBuilder.BROADCAST_KEY) != null ) {
- MonitoringConfiguration monitoringConfiguration = new MonitoringConfiguration();
- monitoringConfiguration.setBroadcastURI((String)streamConfig.get(LocalStreamBuilder.BROADCAST_KEY));
- if(streamConfig.get(LocalStreamBuilder.BROADCAST_INTERVAL_KEY) != null)
- monitoringConfiguration.setMonitoringBroadcastIntervalMs(Long.parseLong((String)streamConfig.get(LocalStreamBuilder.BROADCAST_INTERVAL_KEY)));
- this.streamConfig.setMonitoring(monitoringConfiguration);
- }
- }
-
- public LocalStreamBuilder(LocalRuntimeConfiguration streamConfig) {
- this.streamConfig = streamConfig;
- this.providers = new HashMap<>();
- this.components = new HashMap<>();
- this.totalTasks = 0;
- this.monitorTasks = 0;
- this.futures = new HashMap<>();
- }
-
- public void prepare() {
- this.streamIdentifier = streamConfig.getIdentifier();
- this.streamConfig.setStartedAt(startedAt.getMillis());
- final LocalStreamBuilder self = this;
- this.shutdownHook = new Thread(() -> {
- LOGGER.debug("Shutdown hook received. Beginning shutdown");
- self.stopInternal(true);
- });
- this.useDeprecatedMonitors = false;
- this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig.getMonitoring());
- }
-
- public void setUseDeprecatedMonitors(boolean useDeprecatedMonitors) {
- this.useDeprecatedMonitors = useDeprecatedMonitors;
- }
-
- @Override
- public StreamBuilder newPerpetualStream(String id, StreamsProvider provider) {
- validateId(id);
- this.providers.put(id, new StreamComponent(id, provider, true, streamConfig));
- ++this.totalTasks;
- if(this.useDeprecatedMonitors && provider instanceof DatumStatusCountable )
- ++this.monitorTasks;
- return this;
- }
-
- @Override
- public StreamBuilder newReadCurrentStream(String id, StreamsProvider provider) {
- validateId(id);
- this.providers.put(id, new StreamComponent(id, provider, false, streamConfig));
- ++this.totalTasks;
- if(this.useDeprecatedMonitors && provider instanceof DatumStatusCountable )
- ++this.monitorTasks;
- return this;
- }
-
- @Override
- public StreamBuilder newReadNewStream(String id, StreamsProvider provider, BigInteger sequence) {
- validateId(id);
- this.providers.put(id, new StreamComponent(id, provider, sequence, streamConfig));
- ++this.totalTasks;
- if(this.useDeprecatedMonitors && provider instanceof DatumStatusCountable )
- ++this.monitorTasks;
- return this;
- }
-
- @Override
- public StreamBuilder newReadRangeStream(String id, StreamsProvider provider, DateTime start, DateTime end) {
- validateId(id);
- this.providers.put(id, new StreamComponent(id, provider, start, end, streamConfig));
- ++this.totalTasks;
- if(this.useDeprecatedMonitors && provider instanceof DatumStatusCountable )
- ++this.monitorTasks;
- return this;
- }
-
- @Override
- public StreamBuilder setStreamsConfiguration(StreamsConfiguration configuration) {
- streamConfig = StreamsJacksonMapper.getInstance().convertValue(configuration, LocalRuntimeConfiguration.class);
- return this;
- }
-
- @Override
- public StreamsConfiguration getStreamsConfiguration() {
- return StreamsJacksonMapper.getInstance().convertValue(streamConfig, StreamsConfiguration.class);
- }
-
- @Override
- public StreamBuilder addStreamsProcessor(String id, StreamsProcessor processor, int numTasks, String... inBoundIds) {
- validateId(id);
- StreamComponent comp = new StreamComponent(id, processor, new ThroughputQueue<>(this.maxQueueCapacity, id, streamIdentifier, startedAt.getMillis()), numTasks, streamConfig);
- this.components.put(id, comp);
- connectToOtherComponents(inBoundIds, comp);
- this.totalTasks += numTasks;
- if(this.useDeprecatedMonitors && processor instanceof DatumStatusCountable )
- ++this.monitorTasks;
- return this;
- }
-
- @Override
- public StreamBuilder addStreamsPersistWriter(String id, StreamsPersistWriter writer, int numTasks, String... inBoundIds) {
- validateId(id);
- StreamComponent comp = new StreamComponent(id, writer, new ThroughputQueue<>(this.maxQueueCapacity, id, streamIdentifier, startedAt.getMillis()), numTasks, streamConfig);
- this.components.put(id, comp);
- connectToOtherComponents(inBoundIds, comp);
- this.totalTasks += numTasks;
- if(this.useDeprecatedMonitors && writer instanceof DatumStatusCountable )
- ++this.monitorTasks;
- return this;
- }
-
- /**
- * Runs the data stream in the this JVM and blocks till completion.
- */
- @Override
- public void start() {
- prepare();
- attachShutdownHandler();
- boolean isRunning = true;
- this.executor = new ShutdownStreamOnUnhandleThrowableThreadPoolExecutor(this.totalTasks, this);
- this.monitor = Executors.newCachedThreadPool();
- Map<String, StreamsProviderTask> provTasks = new HashMap<>();
- tasks = new HashMap<>();
- boolean forcedShutDown = false;
-
- try {
- if (this.useDeprecatedMonitors) {
- monitorThread = new LocalStreamProcessMonitorThread(executor, 10);
- this.monitor.submit(monitorThread);
- }
- setupComponentTasks(tasks);
- setupProviderTasks(provTasks);
- LOGGER.info("Started stream with {} components", tasks.size());
- while(isRunning) {
- Uninterruptibles.sleepUninterruptibly(streamConfig.getShutdownCheckDelay(), TimeUnit.MILLISECONDS);
- isRunning = false;
- for(StreamsProviderTask task : provTasks.values()) {
- isRunning = isRunning || task.isRunning();
- }
- for(StreamComponent task: components.values()) {
- boolean tasksRunning = false;
- for(StreamsTask t : task.getStreamsTasks()) {
- if(t instanceof BaseStreamsTask) {
- tasksRunning = tasksRunning || t.isRunning();
- }
- }
- isRunning = isRunning || (tasksRunning && task.getInBoundQueue().size() > 0);
- }
- if(isRunning) {
- Uninterruptibles.sleepUninterruptibly(streamConfig.getShutdownCheckInterval(), TimeUnit.MILLISECONDS);
- }
- }
- LOGGER.info("Components are no longer running or timed out");
- } catch (Exception e){
- LOGGER.warn("Runtime exception. Beginning shutdown");
- forcedShutDown = true;
- } finally{
- LOGGER.info("Stream has completed, pausing @ {}", System.currentTimeMillis());
- Uninterruptibles.sleepUninterruptibly(streamConfig.getShutdownPauseMs(), TimeUnit.MILLISECONDS);
- LOGGER.info("Stream has completed, shutting down @ {}", System.currentTimeMillis());
- stopInternal(forcedShutDown);
- }
-
- }
-
- private void attachShutdownHandler() {
- LOGGER.debug("Attaching shutdown handler");
- Runtime.getRuntime().addShutdownHook(shutdownHook);
- }
-
- private void detachShutdownHandler() {
- LOGGER.debug("Detaching shutdown handler");
- Runtime.getRuntime().removeShutdownHook(shutdownHook);
- }
-
- protected void forceShutdown(Map<String, List<StreamsTask>> streamsTasks) {
- LOGGER.debug("Shutdown failed. Forcing shutdown");
- for(List<StreamsTask> tasks : streamsTasks.values()) {
- for(StreamsTask task : tasks) {
- task.stopTask();
- if(task.isWaiting()) {
- this.futures.get(task).cancel(true);
- }
- }
- }
- this.executor.shutdown();
- this.monitor.shutdown();
- try {
- if(!this.executor.awaitTermination(streamConfig.getExecutorShutdownPauseMs(), TimeUnit.MILLISECONDS)){
- this.executor.shutdownNow();
- }
- if(!this.monitor.awaitTermination(streamConfig.getMonitorShutdownPauseMs(), TimeUnit.MILLISECONDS)){
- this.monitor.shutdownNow();
- }
- }catch (InterruptedException ie) {
- this.executor.shutdownNow();
- this.monitor.shutdownNow();
- throw new RuntimeException(ie);
- }
- }
-
- protected void shutdown(Map<String, List<StreamsTask>> streamsTasks) throws InterruptedException {
- LOGGER.info("Attempting to shutdown tasks");
- if (this.monitorThread != null) {
- this.monitorThread.shutdown();
- }
- this.executor.shutdown();
- //complete stream shut down gracfully
- for(StreamComponent prov : this.providers.values()) {
- shutDownTask(prov, streamsTasks);
- }
- //need to make this configurable
- if(!this.executor.awaitTermination(streamConfig.getExecutorShutdownWaitMs(), TimeUnit.MILLISECONDS)) { // all threads should have terminated already.
- this.executor.shutdownNow();
- this.executor.awaitTermination(streamConfig.getExecutorShutdownWaitMs(), TimeUnit.MILLISECONDS);
- }
- if(!this.monitor.awaitTermination(streamConfig.getMonitorShutdownWaitMs(), TimeUnit.MILLISECONDS)) { // all threads should have terminated already.
- this.monitor.shutdownNow();
- this.monitor.awaitTermination(streamConfig.getMonitorShutdownWaitMs(), TimeUnit.MILLISECONDS);
- }
- }
-
- protected void setupProviderTasks(Map<String, StreamsProviderTask> provTasks) {
- for(StreamComponent prov : this.providers.values()) {
- StreamsTask task = prov.createConnectedTask(getTimeout());
- task.setStreamConfig(this.streamConfig);
- StreamsTaskCounter counter = new StreamsTaskCounter(prov.getId(), streamIdentifier, startedAt.getMillis());
- task.setStreamsTaskCounter(counter);
- this.executor.submit(task);
- provTasks.put(prov.getId(), (StreamsProviderTask) task);
- if(this.useDeprecatedMonitors && prov.isOperationCountable() ) {
- this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) prov.getOperation(), 10));
- this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) task, 10));
- }
- }
- }
-
- protected void setupComponentTasks(Map<String, List<StreamsTask>> streamsTasks) {
- for(StreamComponent comp : this.components.values()) {
- int tasks = comp.getNumTasks();
- List<StreamsTask> compTasks = new LinkedList<>();
- StreamsTaskCounter counter = new StreamsTaskCounter(comp.getId(), streamIdentifier, startedAt.getMillis());
- for(int i=0; i < tasks; ++i) {
- StreamsTask task = comp.createConnectedTask(getTimeout());
- task.setStreamsTaskCounter(counter);
- task.setStreamConfig(this.streamConfig);
- this.futures.put(task, this.executor.submit(task));
- compTasks.add(task);
- if(this.useDeprecatedMonitors && comp.isOperationCountable() ) {
- this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) comp.getOperation(), 10));
- this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) task, 10));
- }
- this.monitor.submit(broadcastMonitor);
- }
- streamsTasks.put(comp.getId(), compTasks);
- }
- }
-
- /**
- * Shutsdown the running tasks in sudo depth first search kind of way. Checks that the upstream components have
- * finished running before shutting down. Waits till inbound queue is empty to shutdown.
- * @param comp StreamComponent to shut down.
- * @param streamTasks the list of non-StreamsProvider tasks for this stream.
- * @throws InterruptedException
- */
- private void shutDownTask(StreamComponent comp, Map<String, List<StreamsTask>> streamTasks) throws InterruptedException {
- List<StreamsTask> tasks = streamTasks.get(comp.getId());
- if(tasks != null) { //not a StreamProvider
- boolean parentsShutDown = true;
- for(StreamComponent parent : comp.getUpStreamComponents()) {
- List<StreamsTask> parentTasks = streamTasks.get(parent.getId());
- //if parentTask == null, its a provider and is not running anymore
- if(parentTasks != null) {
- for(StreamsTask task : parentTasks) {
- parentsShutDown = parentsShutDown && !task.isRunning();
- }
- }
- }
- if(parentsShutDown) {
- for(StreamsTask task : tasks) {
- task.stopTask();
- if(task.isWaiting()) {
- this.futures.get(task).cancel(true); // no data to process, interrupt block queue
- }
- }
- for(StreamsTask task : tasks) {
- int count = 0;
- while(count < streamConfig.getTaskTimeoutMs() / 1000 && task.isRunning()) {
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
- count++;
- }
-
- if(task.isRunning()) {
- LOGGER.warn("Task {} failed to terminate in allotted timeframe", task.toString());
- }
- }
- }
- }
- Collection<StreamComponent> children = comp.getDownStreamComponents();
- if(children != null) {
- for(StreamComponent child : comp.getDownStreamComponents()) {
- shutDownTask(child, streamTasks);
- }
- }
- }
-
- /**
- * NOT IMPLEMENTED.
- */
- @Override
- public void stop() {
- stopInternal(false);
- }
-
-
-
- protected void stopInternal(boolean systemExiting) {
- try {
- shutdown(tasks);
- } catch (Exception e) {
- LOGGER.error("Exception while trying to shutdown Stream: {}", e);
- forceShutdown(tasks);
- } finally {
- try {
- if(!systemExiting) {
- detachShutdownHandler();
- }
- } catch( Throwable e3 ) {
- LOGGER.error("StopInternal caught Throwable: {}", e3);
- System.exit(1);
- }
- }
- }
-
- private void connectToOtherComponents(String[] conntectToIds, StreamComponent toBeConnected) {
- for(String id : conntectToIds) {
- StreamComponent upStream;
- if(this.providers.containsKey(id)) {
- upStream = this.providers.get(id);
- }
- else if(this.components.containsKey(id)) {
- upStream = this.components.get(id);
- }
- else {
- throw new InvalidStreamException("Cannot connect to id, "+id+", because id does not exist.");
- }
- upStream.addOutBoundQueue(toBeConnected, toBeConnected.getInBoundQueue());
- toBeConnected.addInboundQueue(upStream);
- }
- }
-
- private void validateId(String id) {
- if(this.providers.containsKey(id) || this.components.containsKey(id)) {
- throw new InvalidStreamException("Duplicate id. "+id+" is already assigned to another component");
- } else if(id.contains(":")) {
- throw new InvalidStreamException("Invalid character, ':', in component id : "+id);
- }
- }
-
- protected int getTimeout() {
- //Set the timeout of it is configured, otherwise signal downstream components to use their default
- return streamConfig.getProviderTimeoutMs().intValue();
- }
-
- private LocalRuntimeConfiguration convertConfiguration(Map<String, Object> streamConfig) {
- LocalRuntimeConfiguration config = new LocalRuntimeConfiguration();
- if( streamConfig != null ) {
- for( Map.Entry<String, Object> item : streamConfig.entrySet() ) {
- config.setAdditionalProperty(item.getKey(), item.getValue());
- }
- }
- return config;
- }
-
-}
\ No newline at end of file
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
deleted file mode 100644
index e25767e..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.builders;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.core.DatumStatusCountable;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsOperation;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.local.tasks.StreamsPersistWriterTask;
-import org.apache.streams.local.tasks.StreamsProcessorTask;
-import org.apache.streams.local.tasks.StreamsProviderTask;
-import org.apache.streams.local.tasks.StreamsTask;
-import org.apache.streams.util.SerializationUtil;
-
-import org.joda.time.DateTime;
-
-import java.io.Serializable;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-
-/**
- * Stores the implementations of {@link org.apache.streams.core.StreamsOperation}, the StreamsOperations it is connected
- * to and the necessary metadata to construct a data stream.
- */
-public class StreamComponent implements Serializable {
-
- private static final int START = 1;
- private static final int END = 2;
-
- private String id;
- private Set<StreamComponent> inBound;
- private Map<StreamComponent, BlockingQueue<StreamsDatum>> outBound;
- private BlockingQueue<StreamsDatum> inQueue;
- private StreamsProvider provider;
- private StreamsProcessor processor;
- private StreamsPersistWriter writer;
- private DateTime[] dateRange;
- private BigInteger sequence;
- private int numTasks = 1;
- private boolean perpetual;
-
- private List<StreamsTask> tasks;
-
- private StreamsConfiguration streamConfig;
-
- /**
- *
- * @param id
- * @param provider
- */
- public StreamComponent(String id, StreamsProvider provider, boolean perpetual, StreamsConfiguration streamConfig) {
- this.id = id;
- this.provider = provider;
- this.perpetual = perpetual;
- this.streamConfig = streamConfig;
- initializePrivateVariables();
- }
-
- /**
- *
- * @param id
- * @param provider
- * @param start
- * @param end
- */
- public StreamComponent(String id, StreamsProvider provider, DateTime start, DateTime end, StreamsConfiguration streamConfig) {
- this.id = id;
- this.provider = provider;
- this.dateRange = new DateTime[2];
- this.dateRange[START] = start;
- this.dateRange[END] = end;
- this.streamConfig = streamConfig;
- initializePrivateVariables();
- }
-
-
- /**
- *
- * @param id
- * @param provider
- * @param sequence
- */
- public StreamComponent(String id, StreamsProvider provider, BigInteger sequence, StreamsConfiguration streamConfig) {
- this.id = id;
- this.provider = provider;
- this.sequence = sequence;
- this.streamConfig = streamConfig;
- }
-
- /**
- *
- * @param id
- * @param processor
- * @param inQueue
- * @param numTasks
- */
- public StreamComponent(String id, StreamsProcessor processor, BlockingQueue<StreamsDatum> inQueue, int numTasks, StreamsConfiguration streamConfig) {
- this.id = id;
- this.processor = processor;
- this.inQueue = inQueue;
- this.numTasks = numTasks;
- this.streamConfig = streamConfig;
- initializePrivateVariables();
- }
-
- /**
- *
- * @param id
- * @param writer
- * @param inQueue
- * @param numTasks
- */
- public StreamComponent(String id, StreamsPersistWriter writer, BlockingQueue<StreamsDatum> inQueue, int numTasks, StreamsConfiguration streamConfig) {
- this.id = id;
- this.writer = writer;
- this.inQueue = inQueue;
- this.numTasks = numTasks;
- this.streamConfig = streamConfig;
- initializePrivateVariables();
- }
-
- private void initializePrivateVariables() {
- this.inBound = new HashSet<>();
- this.outBound = new HashMap<>();
- this.tasks = new ArrayList<>();
- }
-
- /**
- * Add an outbound queue for this component. The queue should be an inbound queue of a downstream component.
- * @param component the component that this supplying their inbound queue
- * @param queue the queue to to put post processed/provided datums on
- */
- public void addOutBoundQueue(StreamComponent component, BlockingQueue<StreamsDatum> queue) {
- this.outBound.put(component, queue);
- }
-
- /**
- * Add a component that supplies data through the inbound queue.
- * @param component that supplies data through the inbound queue
- */
- public void addInboundQueue(StreamComponent component) {
- this.inBound.add(component);
- }
-
- /**
- * The components that are immediately downstream of this component (aka child nodes)
- * @return Collection of child nodes of this component
- */
- public Collection<StreamComponent> getDownStreamComponents() {
- return this.outBound.keySet();
- }
-
- /**
- * The components that are immediately upstream of this component (aka parent nodes)
- * @return Collection of parent nodes of this component
- */
- public Collection<StreamComponent> getUpStreamComponents() {
- return this.inBound;
- }
-
- /**
- * The inbound queue for this component
- * @return inbound queue
- */
- public BlockingQueue<StreamsDatum> getInBoundQueue() {
- return this.inQueue;
- }
-
- /**
- * The number of tasks this to run this component
- * @return
- */
- public int getNumTasks() {
- return this.numTasks;
- }
-
- /**
- * Creates a {@link org.apache.streams.local.tasks.StreamsTask} that is running a clone of this component whose
- * inbound and outbound queues are appropriately connected to the parent and child nodes.
- *
- * @return StreamsTask for this component
- * @param timeout The timeout to use in milliseconds for any tasks that support configurable timeout
- */
- public StreamsTask createConnectedTask(int timeout) {
- StreamsTask task;
- if(this.processor != null) {
- if(this.numTasks > 1) {
- task = new StreamsProcessorTask(SerializationUtil.cloneBySerialization(this.processor), streamConfig);
- task.addInputQueue(this.inQueue);
- for(BlockingQueue<StreamsDatum> q : this.outBound.values()) {
- task.addOutputQueue(q);
- }
- } else {
- task = new StreamsProcessorTask(this.processor, streamConfig);
- task.addInputQueue(this.inQueue);
- for(BlockingQueue<StreamsDatum> q : this.outBound.values()) {
- task.addOutputQueue(q);
- }
- }
- }
- else if(this.writer != null) {
- if(this.numTasks > 1) {
- task = new StreamsPersistWriterTask(SerializationUtil.cloneBySerialization(this.writer), streamConfig);
- task.addInputQueue(this.inQueue);
- } else {
- task = new StreamsPersistWriterTask(this.writer, streamConfig);
- task.addInputQueue(this.inQueue);
- }
- }
- else if(this.provider != null) {
- StreamsProvider prov;
- if(this.numTasks > 1) {
- prov = SerializationUtil.cloneBySerialization(this.provider);
- } else {
- prov = this.provider;
- }
- if(this.dateRange == null && this.sequence == null)
- task = new StreamsProviderTask(prov, this.perpetual, streamConfig);
- else if(this.sequence != null)
- task = new StreamsProviderTask(prov, this.sequence, streamConfig);
- else
- task = new StreamsProviderTask(prov, this.dateRange[0], this.dateRange[1], streamConfig);
- //Adjust the timeout if necessary
- if(timeout != 0) {
- ((StreamsProviderTask)task).setTimeout(timeout);
- }
- for(BlockingQueue<StreamsDatum> q : this.outBound.values()) {
- task.addOutputQueue(q);
- }
- }
- else {
- throw new InvalidStreamException("Underlying StreamComponoent was NULL.");
- }
-
- if(task != null) {
- tasks.add(task);
- }
-
- return task;
- }
-
- public List<StreamsTask> getStreamsTasks() {
- return this.tasks;
- }
-
- /**
- * The unique of this component
- * @return
- */
- public String getId() {
- return this.id;
- }
-
- @Override
- public int hashCode() {
- return this.id.hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- return o instanceof StreamComponent && this.id.equals(((StreamComponent) o).id);
- }
-
- protected StreamsOperation getOperation() {
- if(this.processor != null) {
- return this.processor;
- }
- else if(this.writer != null) {
- return this.writer;
- }
- else if(this.provider != null) {
- return this.provider;
- }
- else {
- throw new InvalidStreamException("Underlying StreamComponoent was NULL.");
- }
- }
-
- @Deprecated
- protected boolean isOperationCountable() {
- return getOperation() instanceof DatumStatusCountable;
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/counters/DatumStatusCounter.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/counters/DatumStatusCounter.java
deleted file mode 100644
index e2884d0..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/counters/DatumStatusCounter.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.streams.local.counters;
-
-import org.apache.streams.local.builders.LocalStreamBuilder;
-import org.apache.streams.util.ComponentUtils;
-
-import net.jcip.annotations.ThreadSafe;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- *
- */
-@ThreadSafe
-public class DatumStatusCounter implements DatumStatusCounterMXBean{
-
- public static final String NAME_TEMPLATE = "org.apache.streams.local:type=DatumCounter,name=%s,identifier=%s,startedAt=%s";
- private static final Logger LOGGER = LoggerFactory.getLogger(DatumStatusCounter.class);
-
- private AtomicLong failed;
- private AtomicLong passed;
-
- public DatumStatusCounter(String id) {
- this(id, LocalStreamBuilder.DEFAULT_STREAM_IDENTIFIER, -1);
- }
-
- public DatumStatusCounter(String id, String streamIdentifier, long startedAt) {
- this.failed = new AtomicLong(0);
- this.passed = new AtomicLong(0);
- ComponentUtils.registerLocalMBean(String.format(NAME_TEMPLATE, id, streamIdentifier, startedAt), this);
- }
-
- public void incrementFailedCount() {
- this.incrementFailedCount(1);
- }
-
- public void incrementFailedCount(long delta) {
- this.failed.addAndGet(delta);
- }
-
- public void incrementPassedCount() {
- this.incrementPassedCount(1);
- }
-
- public void incrementPassedCount(long delta) {
- this.passed.addAndGet(delta);
- }
-
-
- @Override
- public double getFailRate() {
- double failed = this.failed.get();
- double passed = this.passed.get();
- if(failed == 0.0 && passed == 0) {
- return 0.0;
- }
- return failed / (passed + failed);
- }
-
- @Override
- public long getNumFailed() {
- return this.failed.get();
- }
-
- @Override
- public long getNumPassed() {
- return this.passed.get();
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/counters/DatumStatusCounterMXBean.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/counters/DatumStatusCounterMXBean.java
deleted file mode 100644
index 3a318a8..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/counters/DatumStatusCounterMXBean.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.streams.local.counters;
-
-/**
- *
- */
-public interface DatumStatusCounterMXBean {
-
- /**
- * Get number of failed datums
- * @return number of failed datums
- */
- public long getNumFailed();
-
- /**
- * Get number of passed datums
- * @return number of passed datums
- */
- public long getNumPassed();
-
- /**
- * Get the failure rate. Calculated by num failed divided by (num passed + num failed)
- * @return the failure rate
- */
- public double getFailRate();
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/counters/StreamsTaskCounter.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/counters/StreamsTaskCounter.java
deleted file mode 100644
index de37f1b..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/counters/StreamsTaskCounter.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.streams.local.counters;
-
-import org.apache.streams.local.builders.LocalStreamBuilder;
-import org.apache.streams.util.ComponentUtils;
-
-import net.jcip.annotations.GuardedBy;
-import net.jcip.annotations.ThreadSafe;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- *
- */
-@ThreadSafe
-public class StreamsTaskCounter implements StreamsTaskCounterMXBean {
-
- public static final String NAME_TEMPLATE = "org.apache.streams.local:type=StreamsTaskCounter,name=%s,identifier=%s,startedAt=%s";
- private static final Logger LOGGER = LoggerFactory.getLogger(StreamsTaskCounter.class);
-
- private AtomicLong emitted;
- private AtomicLong received;
- private AtomicLong errors;
- private AtomicLong totalTime;
- @GuardedBy("this")
- private volatile long maxTime;
-
- /**
- *
- * @param id
- */
- public StreamsTaskCounter(String id) {
- this(id, LocalStreamBuilder.DEFAULT_STREAM_IDENTIFIER, -1);
- }
-
- /**
- *
- * @param id
- */
- public StreamsTaskCounter(String id, String streamId, long startedAt) {
- this.emitted = new AtomicLong(0);
- this.received = new AtomicLong(0);
- this.errors = new AtomicLong(0);
- this.totalTime = new AtomicLong(0);
- this.maxTime = -1;
- ComponentUtils.registerLocalMBean(String.format(NAME_TEMPLATE, id, streamId, startedAt), this);
- }
-
- /**
- * Increment emitted count
- */
- public void incrementEmittedCount() {
- this.incrementEmittedCount(1);
- }
-
- /**
- * Increment emitted count
- * @param delta
- */
- public void incrementEmittedCount(long delta) {
- this.emitted.addAndGet(delta);
- }
-
- /**
- * Increment error count
- */
- public void incrementErrorCount() {
- this.incrementErrorCount(1);
- }
-
- /**
- * Increment error count
- * @param delta
- */
- public void incrementErrorCount(long delta) {
- this.errors.addAndGet(delta);
- }
-
- /**
- * Increment received count
- */
- public void incrementReceivedCount() {
- this.incrementReceivedCount(1);
- }
-
- /**
- * Increment received count
- * @param delta
- */
- public void incrementReceivedCount(long delta) {
- this.received.addAndGet(delta);
- }
-
- /**
- * Add the time it takes to process a single datum in milliseconds
- * @param processTime
- */
- public void addTime(long processTime) {
- synchronized (this) {
- if(processTime > this.maxTime) {
- this.maxTime = processTime;
- }
- }
- this.totalTime.addAndGet(processTime);
- }
-
- @Override
- public double getErrorRate() {
- if(this.received.get() == 0) {
- return 0.0;
- }
- return (double) this.errors.get() / (double) this.received.get();
- }
-
- @Override
- public long getNumEmitted() {
- return this.emitted.get();
- }
-
- @Override
- public long getNumReceived() {
- return this.received.get();
- }
-
- @Override
- public long getNumUnhandledErrors() {
- return this.errors.get();
- }
-
- @Override
- public double getAvgTime() {
- long rec = this.received.get();
- long emit = this.emitted.get();
- if(rec == 0 && emit == 0 ) {
- return 0.0;
- } else if( rec == 0) { //provider instance
- return this.totalTime.get() / (double) emit;
- } else {
- return this.totalTime.get() / ((double) this.received.get() - this.errors.get());
- }
- }
-
- @Override
- public long getMaxTime() {
- return this.maxTime;
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/counters/StreamsTaskCounterMXBean.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/counters/StreamsTaskCounterMXBean.java
deleted file mode 100644
index 062eb04..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/counters/StreamsTaskCounterMXBean.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.streams.local.counters;
-
-/**
- *
- */
-public interface StreamsTaskCounterMXBean {
-
- /**
- * Get the error rate of the streams process calculated by the number of errors not handled by the {@link org.apache.streams.local.tasks.StreamsTask}
- * divided by the number of datums received.
- * @return error rate
- */
- public double getErrorRate();
-
- /**
- * Get the number of {@link org.apache.streams.core.StreamsDatum}s emitted by the streams process
- * @return number of emitted datums
- */
- public long getNumEmitted();
-
- /**
- * Get the number of {@link org.apache.streams.core.StreamsDatum}s received by the streams process
- * @return number of received datums
- */
- public long getNumReceived();
-
- /**
- * Get the number of errors that the process had to catch because the executing Provider/Processor/Writer did not
- * catch and handle the exception
- * @return number of handled errors
- */
- public long getNumUnhandledErrors();
-
- /**
- * Returns the average time in milliseconds it takes the task to readCurrent, process, or write to return.
- * @return
- */
- public double getAvgTime();
-
- /**
- * Returns the max time in milliseconds it takes the task to readCurrent, process, or write to return.
- * @return
- */
- public long getMaxTime();
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/executors/ShutdownStreamOnUnhandleThrowableThreadPoolExecutor.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/executors/ShutdownStreamOnUnhandleThrowableThreadPoolExecutor.java
deleted file mode 100644
index 64dd72d..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/executors/ShutdownStreamOnUnhandleThrowableThreadPoolExecutor.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.executors;
-
-import org.apache.streams.local.builders.LocalStreamBuilder;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * A fixed ThreadPoolExecutor that will shutdown a stream upon a thread ending execution due to an unhandled throwable.
- * @see java.util.concurrent.ThreadPoolExecutor
- */
-public class ShutdownStreamOnUnhandleThrowableThreadPoolExecutor extends ThreadPoolExecutor {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ShutdownStreamOnUnhandleThrowableThreadPoolExecutor.class);
-
- private LocalStreamBuilder streamBuilder;
- private volatile boolean isStoped;
-
- /**
- * Creates a fixed size thread pool where corePoolSize & maximumPoolSize equal numThreads with an unbounded queue.
- * @param numThreads number of threads in pool
- * @param streamBuilder streambuilder to call {@link org.apache.streams.core.StreamBuilder#stop()} on upon receiving an unhandled throwable
- */
- public ShutdownStreamOnUnhandleThrowableThreadPoolExecutor(int numThreads, LocalStreamBuilder streamBuilder) {
- super(numThreads, numThreads, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
- this.streamBuilder = streamBuilder;
- this.isStoped = false;
- }
-
- @Override
- protected void afterExecute(Runnable r, Throwable t) {
- if(t != null) {
- LOGGER.error("Runnable, {}, exited with an unhandled throwable! : {}", r.getClass(), t);
- LOGGER.error("Attempting to shut down stream.");
- synchronized (this) {
- if (!this.isStoped) {
- this.isStoped = true;
- this.streamBuilder.stop();
- }
- }
- } else {
- LOGGER.trace("Runnable, {}, finished executing.", r.getClass());
- }
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/queues/ThroughputQueue.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/queues/ThroughputQueue.java
deleted file mode 100644
index aae12a0..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/queues/ThroughputQueue.java
+++ /dev/null
@@ -1,486 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.streams.local.queues;
-
-import org.apache.streams.local.builders.LocalStreamBuilder;
-
-import org.apache.commons.lang.NotImplementedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.management.ManagementFactory;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import javax.management.InstanceAlreadyExistsException;
-import javax.management.MBeanRegistrationException;
-import javax.management.MBeanServer;
-import javax.management.MalformedObjectNameException;
-import javax.management.NotCompliantMBeanException;
-import javax.management.ObjectName;
-
-/**
- * A {@link java.util.concurrent.BlockingQueue} implementation that allows the measure measurement of how
- * data flows through the queue. Is also a {@code MBean} so the flow statistics can be viewed through
- * JMX. Registration of the bean happens whenever a constructor receives a non-null id.
- * <p/>
- * !!! Warning !!!
- * Only the necessary methods for the local streams runtime are implemented. All other methods throw a
- * {@link org.apache.commons.lang.NotImplementedException}.
- */
-public class ThroughputQueue<E> implements BlockingQueue<E>, ThroughputQueueMXBean {
-
- public static final String NAME_TEMPLATE = "org.apache.streams.local:type=ThroughputQueue,name=%s,identifier=%s,startedAt=%s";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ThroughputQueue.class);
-
- private BlockingQueue<ThroughputElement<E>> underlyingQueue;
- private AtomicLong elementsAdded;
- private AtomicLong elementsRemoved;
- private AtomicLong startTime;
- private AtomicLong totalQueueTime;
- private long maxQueuedTime;
- private volatile boolean active;
- private ReadWriteLock maxQueueTimeLock;
-
- /**
- * Creates an unbounded, unregistered {@code ThroughputQueue}
- */
- public ThroughputQueue() {
- this(-1, null, LocalStreamBuilder.DEFAULT_STREAM_IDENTIFIER, -1);
- }
-
- /**
- *
- * @param streamIdentifier
- * @param startedAt
- */
- public ThroughputQueue(String streamIdentifier, long startedAt) {
- this(-1, null, streamIdentifier, startedAt);
- }
-
- /**
- * Creates a bounded, unregistered {@code ThroughputQueue}
- *
- * @param maxSize maximum capacity of queue, if maxSize < 1 then unbounded
- */
- public ThroughputQueue(int maxSize) {
- this(maxSize, null, LocalStreamBuilder.DEFAULT_STREAM_IDENTIFIER, -1);
- }
-
- /**
- *
- * @param maxSize
- * @param streamIdentifier
- * @param startedAt
- */
- public ThroughputQueue(int maxSize, String streamIdentifier, long startedAt) {
- this(maxSize, null, streamIdentifier, startedAt);
- }
-
- /**
- * Creates an unbounded, registered {@code ThroughputQueue}
- *
- * @param id unique id for this queue to be registered with. if id == NULL then not registered
- */
- public ThroughputQueue(String id) {
- this(-1, id, LocalStreamBuilder.DEFAULT_STREAM_IDENTIFIER, -1);
- }
-
- /**
- *
- * @param id
- * @param streamIdentifier
- * @param startedAt
- */
- public ThroughputQueue(String id, String streamIdentifier, long startedAt) {
- this(-1, id, streamIdentifier, startedAt);
- }
-
- /**
- *
- * @param maxSize
- * @param id
- */
- public ThroughputQueue(int maxSize, String id) {
- this(maxSize, id, LocalStreamBuilder.DEFAULT_STREAM_IDENTIFIER, -1);
-
- }
-
- /**
- * Creates a bounded, registered {@code ThroughputQueue}
- *
- * @param maxSize maximum capacity of queue, if maxSize < 1 then unbounded
- * @param id unique id for this queue to be registered with. if id == NULL then not registered
- */
- public ThroughputQueue(int maxSize, String id, String streamIdentifier, long startedAt) {
- if (maxSize < 1) {
- this.underlyingQueue = new LinkedBlockingQueue<>();
- } else {
- this.underlyingQueue = new LinkedBlockingQueue<>(maxSize);
- }
- this.elementsAdded = new AtomicLong(0);
- this.elementsRemoved = new AtomicLong(0);
- this.startTime = new AtomicLong(-1);
- this.active = false;
- this.maxQueuedTime = 0;
- this.maxQueueTimeLock = new ReentrantReadWriteLock();
- this.totalQueueTime = new AtomicLong(0);
- if (id != null) {
- try {
- ObjectName name = new ObjectName(String.format(NAME_TEMPLATE, id, streamIdentifier, startedAt));
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- mbs.registerMBean(this, name);
- } catch (MalformedObjectNameException | InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException e) {
- LOGGER.error("Failed to register MXBean : {}", e);
- throw new RuntimeException(e);
- }
- }
- }
-
- @Override
- public boolean add(E e) {
- if (this.underlyingQueue.add(new ThroughputElement<E>(e))) {
- internalAddElement();
- return true;
- }
- return false;
- }
-
- @Override
- public boolean offer(E e) {
- if (this.underlyingQueue.offer(new ThroughputElement<E>(e))) {
- internalAddElement();
- return true;
- }
- return false;
- }
-
- @Override
- public void put(E e) throws InterruptedException {
- this.underlyingQueue.put(new ThroughputElement<E>(e));
- internalAddElement();
- }
-
- @Override
- public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException {
- if (this.underlyingQueue.offer(new ThroughputElement<E>(e), timeout, unit)) {
- internalAddElement();
- return true;
- }
- return false;
- }
-
- @Override
- public E take() throws InterruptedException {
- ThroughputElement<E> e = this.underlyingQueue.take();
- internalRemoveElement(e);
- return e.getElement();
- }
-
- @Override
- public E poll(long timeout, TimeUnit unit) throws InterruptedException {
- ThroughputElement<E> e = this.underlyingQueue.poll(timeout, unit);
- if(e != null) {
- internalRemoveElement(e);
- return e.getElement();
- }
- return null;
- }
-
- @Override
- public int remainingCapacity() {
- return this.underlyingQueue.remainingCapacity();
- }
-
- @Override
- public boolean remove(Object o) {
- try {
- return this.underlyingQueue.remove(new ThroughputElement<E>((E) o));
- } catch (ClassCastException cce) {
- return false;
- }
- }
-
- @Override
- public boolean contains(Object o) {
- try {
- return this.underlyingQueue.contains(new ThroughputElement<E>((E) o));
- } catch (ClassCastException cce) {
- return false;
- }
- }
-
- @Override
- public int drainTo(Collection<? super E> c) {
- throw new NotImplementedException();
- }
-
- @Override
- public int drainTo(Collection<? super E> c, int maxElements) {
- throw new NotImplementedException();
- }
-
- @Override
- public E remove() {
- ThroughputElement<E> e = this.underlyingQueue.remove();
- if(e != null) {
- internalRemoveElement(e);
- return e.getElement();
- }
- return null;
- }
-
- @Override
- public E poll() {
- ThroughputElement<E> e = this.underlyingQueue.poll();
- if(e != null) {
- internalRemoveElement(e);
- return e.getElement();
- }
- return null;
- }
-
- @Override
- public E element() {
- throw new NotImplementedException();
- }
-
- @Override
- public E peek() {
- ThroughputElement<E> e = this.underlyingQueue.peek();
- if( e != null) {
- return e.getElement();
- }
- return null;
- }
-
- @Override
- public int size() {
- return this.underlyingQueue.size();
- }
-
- @Override
- public boolean isEmpty() {
- return this.underlyingQueue.isEmpty();
- }
-
- @Override
- public Iterator<E> iterator() {
- throw new NotImplementedException();
- }
-
- @Override
- public Object[] toArray() {
- throw new NotImplementedException();
- }
-
- @Override
- public <T> T[] toArray(T[] a) {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean containsAll(Collection<?> c) {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean addAll(Collection<? extends E> c) {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean removeAll(Collection<?> c) {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean retainAll(Collection<?> c) {
- throw new NotImplementedException();
- }
-
- @Override
- public void clear() {
- throw new NotImplementedException();
- }
-
- @Override
- public long getCurrentSize() {
- return this.elementsAdded.get() - this.elementsRemoved.get();
- }
-
- /**
- * If elements have been removed from the queue or no elements have been added, it returns the average wait time
- * in milliseconds. If elements have been added, but none have been removed, it returns the time waited by the first
- * element in the queue.
- *
- * @return the average wait time in milliseconds
- */
- @Override
- public double getAvgWait() {
- if (this.elementsRemoved.get() == 0) {
- if (this.getCurrentSize() > 0) {
- return this.underlyingQueue.peek().getWaited();
- } else {
- return 0.0;
- }
- } else {
- return (double) this.totalQueueTime.get() / (double) this.elementsRemoved.get();
- }
- }
-
- @Override
- public long getMaxWait() {
- ThroughputElement<E> e = this.underlyingQueue.peek();
- long max = -1;
- try {
- this.maxQueueTimeLock.readLock().lock();
- if (e != null && e.getWaited() > this.maxQueuedTime) {
- max = e.getWaited();
- } else {
- max = this.maxQueuedTime;
- }
- } finally {
- this.maxQueueTimeLock.readLock().unlock();
- }
- return max;
- }
-
- @Override
- public long getRemoved() {
- return this.elementsRemoved.get();
- }
-
- @Override
- public long getAdded() {
- return this.elementsAdded.get();
- }
-
- @Override
- public double getThroughput() {
- if (active) {
- return this.elementsRemoved.get() / ((System.currentTimeMillis() - this.startTime.get()) / 1000.0);
- }
- return 0.0;
- }
-
- /**
- * Handles updating the stats whenever elements are added to the queue
- */
- private void internalAddElement() {
- this.elementsAdded.incrementAndGet();
- synchronized (this) {
- if (!this.active) {
- this.startTime.set(System.currentTimeMillis());
- this.active = true;
- }
- }
- }
-
- /**
- * Handle updating the stats whenever elements are removed from the queue
- * @param e Element removed
- */
- private void internalRemoveElement(ThroughputElement<E> e) {
- if(e != null) {
- this.elementsRemoved.incrementAndGet();
- Long queueTime = e.getWaited();
- this.totalQueueTime.addAndGet(queueTime);
- boolean unlocked = false;
- try {
- this.maxQueueTimeLock.readLock().lock();
- if (this.maxQueuedTime < queueTime) {
- this.maxQueueTimeLock.readLock().unlock();
- unlocked = true;
- try {
- this.maxQueueTimeLock.writeLock().lock();
- this.maxQueuedTime = queueTime;
- } finally {
- this.maxQueueTimeLock.writeLock().unlock();
- }
- }
- } finally {
- if (!unlocked)
- this.maxQueueTimeLock.readLock().unlock();
- }
- }
- }
-
-
- /**
- * Element wrapper to measure time waiting on the queue
- *
- * @param <E>
- */
- private class ThroughputElement<E> {
-
- private long queuedTime;
- private E element;
-
- protected ThroughputElement(E element) {
- this.element = element;
- this.queuedTime = System.currentTimeMillis();
- }
-
- /**
- * Get the time this element has been waiting on the queue.
- * current time - time element was queued
- *
- * @return time this element has been waiting on the queue in milliseconds
- */
- public long getWaited() {
- return System.currentTimeMillis() - this.queuedTime;
- }
-
- /**
- * Get the queued element
- *
- * @return the element
- */
- public E getElement() {
- return this.element;
- }
-
-
- /**
- * Measures equality by the element and ignores the queued time
- * @param obj
- * @return
- */
- @Override
- public boolean equals(Object obj) {
- if(obj instanceof ThroughputElement && obj != null) {
- ThroughputElement that = (ThroughputElement) obj;
- if(that.getElement() == null && this.getElement() == null) {
- return true;
- } else if(that.getElement() != null) {
- return that.getElement().equals(this.getElement());
- } else {
- return false;
- }
- }
- return false;
- }
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/queues/ThroughputQueueMXBean.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/queues/ThroughputQueueMXBean.java
deleted file mode 100644
index 9cc4593..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/queues/ThroughputQueueMXBean.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.streams.local.queues;
-
-/**
- * MXBean capable queue that monitors the throughput of the queue
- */
-public interface ThroughputQueueMXBean {
-
- /**
- * Returns the number of items on the queue.
- * @return number of items on queue
- */
- public long getCurrentSize();
-
- /**
- * Get the average time an item spends in queue in milliseconds
- * @return average time an item spends in queue in milliseconds
- */
- public double getAvgWait();
-
- /**
- * Get the maximum time an item has spent on the queue before being removed from the queue.
- * @return the maximum time an item has spent on the queue
- */
- public long getMaxWait();
-
- /**
- * Get the number of items that have been removed from this queue
- * @return number of items that have been removed from the queue
- */
- public long getRemoved();
-
- /**
- * Get the number of items that have been added to the queue
- * @return number of items that have been added to the queue
- */
- public long getAdded();
-
- /**
- * Get the the throughput of the queue measured by the number of items removed from the queue
- * dived by the time the queue has been active.
- * Active time starts once the first item has been placed on the queue
- * @return throughput of queue. items/sec, items removed / time active
- */
- public double getThroughput();
-
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
deleted file mode 100644
index 24922a1..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.tasks;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.util.SerializationUtil;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * BaseStreamsTask is the primary abstract StreamsTask.
- */
-public abstract class BaseStreamsTask implements StreamsTask {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(BaseStreamsTask.class);
-
- private List<BlockingQueue<StreamsDatum>> inQueues = new ArrayList<>();
- private List<BlockingQueue<StreamsDatum>> outQueues = new LinkedList<>();
- private int inIndex = 0;
- private ObjectMapper mapper;
- protected StreamsConfiguration streamConfig;
-
- public BaseStreamsTask(StreamsConfiguration config) {
- this.mapper = StreamsJacksonMapper.getInstance();
- this.mapper.registerSubtypes(Activity.class);
- if( config != null ) {
- this.streamConfig = config;
- } else {
- this.streamConfig = StreamsConfigurator.detectConfiguration();
- }
- setStartedAt();
- }
-
- @Override
- public void addInputQueue(BlockingQueue<StreamsDatum> inputQueue) {
- this.inQueues.add(inputQueue);
- }
-
- @Override
- public void addOutputQueue(BlockingQueue<StreamsDatum> outputQueue) {
- this.outQueues.add(outputQueue);
- }
-
- @Override
- public List<BlockingQueue<StreamsDatum>> getInputQueues() {
- return this.inQueues;
- }
-
- @Override
- public List<BlockingQueue<StreamsDatum>> getOutputQueues() {
- return this.outQueues;
- }
-
- /**
- * SHOULD NOT BE NECCESARY, WILL REMOVE.
- * Round Robins through input queues to get the next StreamsDatum. If all input queues are empty, it will return null.
- * @return the next StreamsDatum or null if all input queues are empty.
- */
- @Deprecated
- protected StreamsDatum getNextDatum() {
- int startIndex = this.inIndex;
- int index = startIndex;
- StreamsDatum datum;
- do {
- datum = this.inQueues.get(index).poll();
- index = getNextInputQueueIndex();
- } while( datum == null && startIndex != index);
- return datum;
- }
-
- /**
- * Adds a StreamDatum to the outgoing queues. If there are multiple queues, it uses serialization to create
- * clones of the datum and adds a new clone to each queue.
- * @param datum
- */
- protected void addToOutgoingQueue(StreamsDatum datum) throws InterruptedException{
- if(this.outQueues.size() == 1) {
- outQueues.get(0).put(datum);
- }
- else {
- List<BlockingQueue<StreamsDatum>> toOutput = new LinkedList<>(this.outQueues);
- while(!toOutput.isEmpty()) {
- for (BlockingQueue<StreamsDatum> queue : toOutput) {
- StreamsDatum newDatum = cloneStreamsDatum(datum);
- if (newDatum != null) {
- if (queue.offer(newDatum, 500, TimeUnit.MILLISECONDS)) {
- toOutput.remove(queue);
- }
- }
- }
- }
- }
- }
-
- @Override
- public boolean isWaiting() {
- if(this.inQueues == null || this.inQueues.size() == 0) {
- return true;
- }
- boolean empty = true;
- for(Queue queue : this.inQueues) {
- empty = empty && queue.isEmpty();
- }
- return empty;
- }
-
- /**
- * //TODO LOCAL MODE HACK. Need to fix
- * In order for our data streams to ported to other data flow frame works(Storm, Hadoop, Spark, etc) we need to be able to
- * enforce the serialization required by each framework. This needs some thought and design before a final solution is
- * made.
- *
- * In order to be able to copy/clone StreamDatums the orginal idea was to force all StreamsDatums to be java serializable.
- * This was seen as unacceptable for local mode. So until we come up with a solution to enforce serialization and be
- * compatiable across multiple frame works, this hack is in place.
- *
- * If datum.document is Serializable, we use serialization to clone a new copy. If it is not Serializable we attempt
- * different methods using an com.fasterxml.jackson.databind.ObjectMapper to copy/clone the StreamsDatum. If the object
- * is not clonable by these methods, an error is reported to the logging and a NULL object is returned.
- *
- * @param datum
- * @return
- */
- protected StreamsDatum cloneStreamsDatum(StreamsDatum datum) {
- try {
-
- if(datum.document instanceof ObjectNode) {
- return copyMetaData(datum, new StreamsDatum(((ObjectNode) datum.document).deepCopy(), datum.timestamp, datum.sequenceid));
- }
- else if(datum.document instanceof Activity) {
-
- return copyMetaData(datum, new StreamsDatum(this.mapper.readValue(this.mapper.writeValueAsString(datum.document), Activity.class),
- datum.timestamp,
- datum.sequenceid));
- }
-// else if(this.mapper.canSerialize(datum.document.getClass())){
-// return new StreamsDatum(this.mapper.readValue(this.mapper.writeValueAsString(datum.document), datum.document.getClass()),
-// datum.timestamp,
-// datum.sequenceid);
-// }
-
- else if(datum.document instanceof Serializable) {
- return SerializationUtil.cloneBySerialization(datum);
- }
- } catch (Exception e) {
- LOGGER.error("Exception while trying to clone/copy StreamsDatum : {}", e);
- }
- LOGGER.error("Failed to clone/copy StreamsDatum with document of class : {}", datum.document.getClass().getName());
- return null;
- }
-
- private int getNextInputQueueIndex() {
- ++this.inIndex;
- if(this.inIndex >= this.inQueues.size()) {
- this.inIndex = 0;
- }
- return this.inIndex;
- }
-
- private StreamsDatum copyMetaData(StreamsDatum copyFrom, StreamsDatum copyTo) {
- Map<String, Object> fromMeta = copyFrom.getMetadata();
- Map<String, Object> toMeta = copyTo.getMetadata();
- for(String key : fromMeta.keySet()) {
- Object value = fromMeta.get(key);
- if(value instanceof Serializable)
- toMeta.put(key, SerializationUtil.cloneBySerialization(value));
- else //hope for the best - should be serializable
- toMeta.put(key, value);
- }
- return copyTo;
- }
-
- public long getStartedAt() {
- return streamConfig.getStartedAt();
- }
-
- public void setStartedAt() {
- streamConfig.setStartedAt(DateTime.now().getMillis());
- }
-
- public String getStreamIdentifier() {
- return streamConfig.getIdentifier();
- }
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/LocalStreamProcessMonitorThread.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/LocalStreamProcessMonitorThread.java
deleted file mode 100644
index 6df9767..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/LocalStreamProcessMonitorThread.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.tasks;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryUsage;
-import java.util.concurrent.Executor;
-
-@Deprecated
-public class LocalStreamProcessMonitorThread implements StatusCounterMonitorRunnable
-{
- private static final Logger LOGGER = LoggerFactory.getLogger(LocalStreamProcessMonitorThread.class);
-
- private Executor executor;
-
- private int seconds;
-
- private boolean run = true;
-
- public LocalStreamProcessMonitorThread(Executor executor, int delayInSeconds) {
- this.executor = executor;
- this.seconds = delayInSeconds;
- }
-
- @Override
- public void shutdown(){
- this.run = false;
- }
-
- @Override
- public boolean isRunning() {
- return this.run;
- }
-
- @Override
- public void run()
- {
- while(run){
-
- /**
- *
- * Note:
- * Quick class and method to let us see what is going on with the JVM. We need to make sure
- * that everything is running with as little memory as possible. If we are generating a heap
- * overflow, this will be very apparent by the information shown here.
- */
-
- MemoryUsage memoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
-
- String maxMemory = memoryUsage.getMax() == Long.MAX_VALUE ? "NO_LIMIT" :
- humanReadableByteCount(memoryUsage.getMax(), true);
-
- String usedMemory = humanReadableByteCount(memoryUsage.getUsed(), true);
-
- LOGGER.debug("[monitor] Used Memory: {}, Max: {}",
- usedMemory,
- maxMemory);
-
- try
- {
- Thread.sleep(seconds*1000);
- }
- catch (InterruptedException e)
- { }
- }
- }
-
- public String humanReadableByteCount(long bytes, boolean si) {
- int unit = si ? 1000 : 1024;
- if (bytes < unit) return bytes + " B";
- int exp = (int) (Math.log(bytes) / Math.log(unit));
- String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp-1) + (si ? "" : "i");
- return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StatusCounterMonitorRunnable.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StatusCounterMonitorRunnable.java
deleted file mode 100644
index 9dc91d3..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StatusCounterMonitorRunnable.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.tasks;
-
-@Deprecated
-public interface StatusCounterMonitorRunnable extends Runnable {
- void shutdown();
- boolean isRunning();
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StatusCounterMonitorThread.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StatusCounterMonitorThread.java
deleted file mode 100644
index d59f0d6..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StatusCounterMonitorThread.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.tasks;
-
-import org.apache.streams.core.DatumStatusCountable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@Deprecated
-public class StatusCounterMonitorThread implements StatusCounterMonitorRunnable {
- private static final Logger LOGGER = LoggerFactory.getLogger(StatusCounterMonitorThread.class);
-
- private DatumStatusCountable task;
-
- private int seconds;
-
- private boolean run = true;
-
- public StatusCounterMonitorThread(DatumStatusCountable task, int delayInSeconds) {
- this.task = task;
- this.seconds = delayInSeconds;
- }
-
- @Override
- public void shutdown() {
- this.run = false;
- }
-
- @Override
- public boolean isRunning() {
- return this.run;
- }
-
- @Override
- public void run() {
- while(run) {
-
- /**
- *
- * Note:
- * Quick class and method to let us see what is going on with the JVM. We need to make sure
- * that everything is running with as little memory as possible. If we are generating a heap
- * overflow, this will be very apparent by the information shown here.
- */
-
- LOGGER.debug("{}: {} attempted, {} success, {} partial, {} failed, {} total",
- task.getClass(),
- task.getDatumStatusCounter().getAttempted(),
- task.getDatumStatusCounter().getSuccess(),
- task.getDatumStatusCounter().getPartial(),
- task.getDatumStatusCounter().getFail(),
- task.getDatumStatusCounter().getEmitted());
-
- try {
- Thread.sleep(seconds*1000);
- }
- catch (InterruptedException e){
- shutdown();
- }
- }
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
deleted file mode 100644
index 473d2f4..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.tasks;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.local.counters.StreamsTaskCounter;
-
-import org.apache.commons.lang.NotImplementedException;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * NOT USED. When joins/partions are implemented, a similar pattern could be followed. Done only as basic proof
- * of concept.
- * NEEDS TO BE RE-WRITTEN
- */
-@Deprecated
-public class StreamsMergeTask extends BaseStreamsTask {
-
- private AtomicBoolean keepRunning;
- private long sleepTime;
-
- public StreamsMergeTask() {
- this(null);
- }
-
- public StreamsMergeTask(StreamsConfiguration streamConfig) {
- super(streamConfig);
- this.sleepTime = sleepTime;
- this.keepRunning = new AtomicBoolean(true);
- }
-
- @Override
- public void stopTask() {
- this.keepRunning.set(false);
- }
-
- @Override
- public void setStreamConfig(StreamsConfiguration config) {
-
- }
-
- @Override
- public boolean isRunning() {
- return false;
- }
-
- @Override
- public void run() {
- while(this.keepRunning.get()) {
- StreamsDatum datum = super.getNextDatum();
- if(datum != null) {
- try {
- super.addToOutgoingQueue(datum);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
- else {
- try {
- Thread.sleep(this.sleepTime);
- } catch (InterruptedException e) {
- this.keepRunning.set(false);
- }
- }
- }
- }
-
- @Override
- public void setStreamsTaskCounter(StreamsTaskCounter counter) {
- throw new NotImplementedException();
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
deleted file mode 100644
index 5c918b2..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.tasks;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.core.DatumStatus;
-import org.apache.streams.core.DatumStatusCountable;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.core.util.DatumUtils;
-import org.apache.streams.local.counters.StreamsTaskCounter;
-
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- *
- */
-public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumStatusCountable {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsPersistWriterTask.class);
-
- private StreamsPersistWriter writer;
- private AtomicBoolean keepRunning;
- private StreamsConfiguration streamConfig;
- private BlockingQueue<StreamsDatum> inQueue;
- private AtomicBoolean isRunning;
- private AtomicBoolean blocked;
- private StreamsTaskCounter counter;
-
- private DatumStatusCounter statusCounter = new DatumStatusCounter();
-
- @Override
- public DatumStatusCounter getDatumStatusCounter() {
- return this.statusCounter;
- }
-
-
- /**
- * Default constructor. Uses default sleep of 500ms when inbound queue is empty.
- * @param writer writer to execute in task
- */
- public StreamsPersistWriterTask(StreamsPersistWriter writer) {
- this(writer, null);
- }
-
- /**
- *
- * @param writer writer to execute in task
- * @param streamConfig stream config
- */
- public StreamsPersistWriterTask(StreamsPersistWriter writer, StreamsConfiguration streamConfig) {
- super(streamConfig);
- this.streamConfig = super.streamConfig;
- this.writer = writer;
- this.keepRunning = new AtomicBoolean(true);
- this.isRunning = new AtomicBoolean(true);
- this.blocked = new AtomicBoolean(false);
- }
-
- @Override
- public boolean isWaiting() {
- return this.inQueue.isEmpty() && this.blocked.get();
- }
-
- @Override
- public void setStreamConfig(StreamsConfiguration config) {
- this.streamConfig = config;
- }
-
- @Override
- public void addInputQueue(BlockingQueue<StreamsDatum> inputQueue) {
- this.inQueue = inputQueue;
- }
-
- @Override
- public boolean isRunning() {
- return this.isRunning.get();
- }
-
- @Override
- public void run() {
- try {
- this.writer.prepare(this.streamConfig);
- if(this.counter == null) {
- this.counter = new StreamsTaskCounter(this.writer.getClass().getName()+ UUID.randomUUID().toString(), getStreamIdentifier(), getStartedAt());
- }
- while(this.keepRunning.get()) {
- StreamsDatum datum = null;
- try {
- this.blocked.set(true);
- datum = this.inQueue.poll(5, TimeUnit.SECONDS);
- } catch (InterruptedException ie) {
- LOGGER.debug("Received InterruptedException. Shutting down and re-applying interrupt status.");
- this.keepRunning.set(false);
- if(!this.inQueue.isEmpty()) {
- LOGGER.error("Received InteruptedException and input queue still has data, count={}, processor={}",this.inQueue.size(), this.writer.getClass().getName());
- }
- Thread.currentThread().interrupt();
- } finally {
- this.blocked.set(false);
- }
- if(datum != null) {
- this.counter.incrementReceivedCount();
- try {
- long startTime = System.currentTimeMillis();
- this.writer.write(datum);
- this.counter.addTime(System.currentTimeMillis() - startTime);
- statusCounter.incrementStatus(DatumStatus.SUCCESS);
- } catch (Exception e) {
- LOGGER.error("Error writing to persist writer {}", this.writer.getClass().getSimpleName(), e);
- this.keepRunning.set(false); // why do we shutdown on a failed write ?
- statusCounter.incrementStatus(DatumStatus.FAIL);
- DatumUtils.addErrorToMetadata(datum, e, this.writer.getClass());
- this.counter.incrementErrorCount();
- }
- } else { //datums should never be null
- LOGGER.trace("Received null StreamsDatum @ writer : {}", this.writer.getClass().getName());
- }
- }
- Uninterruptibles.sleepUninterruptibly(streamConfig.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- } catch(Throwable e) {
- LOGGER.error("Caught Throwable in Persist Writer {} : {}", this.writer.getClass().getSimpleName(), e);
- } finally {
- Uninterruptibles.sleepUninterruptibly(streamConfig.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- this.writer.cleanUp();
- this.isRunning.set(false);
- }
- }
-
- @Override
- public void stopTask() {
- this.keepRunning.set(false);
- }
-
-
- @Override
- public void addOutputQueue(BlockingQueue<StreamsDatum> outputQueue) {
- throw new UnsupportedOperationException(this.getClass().getName()+" does not support method - setOutputQueue()");
- }
-
- @Override
- public List<BlockingQueue<StreamsDatum>> getInputQueues() {
- List<BlockingQueue<StreamsDatum>> queues = new LinkedList<>();
- queues.add(this.inQueue);
- return queues;
- }
-
- @Override
- public void setStreamsTaskCounter(StreamsTaskCounter counter) {
- this.counter = counter;
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
deleted file mode 100644
index de4594a..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.tasks;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.DatumStatus;
-import org.apache.streams.core.DatumStatusCountable;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.core.util.DatumUtils;
-import org.apache.streams.local.counters.StreamsTaskCounter;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- *
- */
-public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatusCountable {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsProcessorTask.class);
-
-
- private StreamsProcessor processor;
- private AtomicBoolean keepRunning;
- private StreamsConfiguration streamConfig;
- private BlockingQueue<StreamsDatum> inQueue;
- private AtomicBoolean isRunning;
- private AtomicBoolean blocked;
- private StreamsTaskCounter counter;
-
- private DatumStatusCounter statusCounter = new DatumStatusCounter();
-
- @Override
- public DatumStatusCounter getDatumStatusCounter() {
- return this.statusCounter;
- }
-
- /**
- * Default constructor, uses default sleep time of 500ms when inbound queue is empty
- * @param processor process to run in task
- */
- public StreamsProcessorTask(StreamsProcessor processor) {
- this(processor, StreamsConfigurator.detectConfiguration());
- }
-
- /**
- * @param processor
- * @param streamConfig
- */
- public StreamsProcessorTask(StreamsProcessor processor, StreamsConfiguration streamConfig) {
- super(streamConfig);
- this.streamConfig = super.streamConfig;
- this.processor = processor;
- this.keepRunning = new AtomicBoolean(true);
- this.isRunning = new AtomicBoolean(true);
- this.blocked = new AtomicBoolean(true);
- }
-
- @Override
- public boolean isWaiting() {
- return this.inQueue.isEmpty() && this.blocked.get();
- }
-
- @Override
- public void stopTask() {
- this.keepRunning.set(false);
- }
-
- @Override
- public void setStreamConfig(StreamsConfiguration config) {
- this.streamConfig = config;
- }
-
- @Override
- public void addInputQueue(BlockingQueue<StreamsDatum> inputQueue) {
- this.inQueue = inputQueue;
- }
-
- @Override
- public boolean isRunning() {
- return this.isRunning.get();
- }
-
- @Override
- public void run() {
- try {
- this.processor.prepare(this.streamConfig);
- if(this.counter == null) {
- this.counter = new StreamsTaskCounter(this.processor.getClass().getName()+ UUID.randomUUID().toString(), getStreamIdentifier(), getStartedAt());
- }
- while(this.keepRunning.get()) {
- StreamsDatum datum = null;
- try {
- this.blocked.set(true);
- datum = this.inQueue.poll(streamConfig.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
- } catch (InterruptedException ie) {
- LOGGER.debug("Received InteruptedException, shutting down and re-applying interrupt status.");
- this.keepRunning.set(false);
- if(!this.inQueue.isEmpty()) {
- LOGGER.error("Received InteruptedException and input queue still has data, count={}, processor={}",this.inQueue.size(), this.processor.getClass().getName());
- }
- Thread.currentThread().interrupt();
- } finally {
- this.blocked.set(false);
- }
- if(datum != null) {
- this.counter.incrementReceivedCount();
- try {
- long startTime = System.currentTimeMillis();
- List<StreamsDatum> output = this.processor.process(datum);
- this.counter.addTime(System.currentTimeMillis() - startTime);
- if(output != null) {
- for(StreamsDatum outDatum : output) {
- super.addToOutgoingQueue(outDatum);
- this.counter.incrementEmittedCount();
- statusCounter.incrementStatus(DatumStatus.SUCCESS);
- }
- }
- } catch (InterruptedException ie) {
- LOGGER.warn("Received InterruptedException, shutting down and re-applying interrupt status.");
- this.keepRunning.set(false);
- Thread.currentThread().interrupt();
- } catch (Throwable t) {
- this.counter.incrementErrorCount();
- LOGGER.warn("Caught Throwable in processor, {} : {}", this.processor.getClass().getName(), t);
- statusCounter.incrementStatus(DatumStatus.FAIL);
- //Add the error to the metadata, but keep processing
- DatumUtils.addErrorToMetadata(datum, t, this.processor.getClass());
- }
- } else {
- LOGGER.trace("Removed NULL datum from queue at processor : {}", this.processor.getClass().getName());
- }
- }
- } catch(Throwable e) {
- LOGGER.error("Caught Throwable in Processor {}", this.processor.getClass().getSimpleName(), e);
- } finally {
- this.isRunning.set(false);
- this.processor.cleanUp();
- }
- }
-
- @Override
- public List<BlockingQueue<StreamsDatum>> getInputQueues() {
- List<BlockingQueue<StreamsDatum>> queues = new LinkedList<BlockingQueue<StreamsDatum>>();
- queues.add(this.inQueue);
- return queues;
- }
-
- @Override
- public void setStreamsTaskCounter(StreamsTaskCounter counter) {
- this.counter = counter;
- }
-
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
deleted file mode 100644
index 26272ea..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.tasks;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.core.DatumStatus;
-import org.apache.streams.core.DatumStatusCountable;
-import org.apache.streams.core.DatumStatusCounter;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.core.util.DatumUtils;
-import org.apache.streams.local.counters.StreamsTaskCounter;
-
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.math.BigInteger;
-import java.util.Queue;
-import java.util.UUID;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- *
- */
-public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusCountable {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsProviderTask.class);
-
- public DatumStatusCounter getDatumStatusCounter() {
- return this.statusCounter;
- }
-
- private static enum Type {
- PERPETUAL,
- READ_CURRENT,
- READ_NEW,
- READ_RANGE
- }
-
- private static final int START = 0;
- private static final int END = 1;
-
- private StreamsProvider provider;
- private final AtomicBoolean keepRunning = new AtomicBoolean(true);
- private final AtomicBoolean flushing = new AtomicBoolean(false);
- private final AtomicBoolean started = new AtomicBoolean(false);
- private Type type;
- private BigInteger sequence;
- private DateTime[] dateRange;
- private StreamsConfiguration config;
-
- private int timeout;
- private long sleepTime;
- private int zeros = 0;
- private DatumStatusCounter statusCounter = new DatumStatusCounter();
- private StreamsTaskCounter counter;
-
- /**
- * Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readCurrent()}
- * @param provider
- */
- public StreamsProviderTask(StreamsProvider provider, boolean perpetual, StreamsConfiguration streamConfig) {
- super(streamConfig);
- streamConfig = super.streamConfig;
- this.provider = provider;
- if( perpetual )
- this.type = Type.PERPETUAL;
- else
- this.type = Type.READ_CURRENT;
- this.timeout = super.streamConfig.getProviderTimeoutMs().intValue();
- this.sleepTime = streamConfig.getBatchFrequencyMs();
- }
-
- /**
- * Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readNew(BigInteger)}
- * @param provider
- * @param sequence
- */
- public StreamsProviderTask(StreamsProvider provider, BigInteger sequence, StreamsConfiguration streamConfig) {
- super(streamConfig);
- this.provider = provider;
- this.type = Type.READ_NEW;
- this.sequence = sequence;
- this.timeout = streamConfig.getProviderTimeoutMs().intValue();
- this.sleepTime = streamConfig.getBatchFrequencyMs();
- }
-
- /**
- * Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readRange(DateTime,DateTime)}
- * @param provider
- * @param start
- * @param end
- */
- public StreamsProviderTask(StreamsProvider provider, DateTime start, DateTime end, StreamsConfiguration streamConfig) {
- super(streamConfig);
- this.provider = provider;
- this.type = Type.READ_RANGE;
- this.dateRange = new DateTime[2];
- this.dateRange[START] = start;
- this.dateRange[END] = end;
- this.timeout = streamConfig.getProviderTimeoutMs().intValue();
- this.sleepTime = streamConfig.getBatchFrequencyMs();
- }
-
- public void setTimeout(int timeout) {
- this.timeout = timeout;
- }
-
- public void setSleepTime(long sleepTime) {
- this.sleepTime = sleepTime;
- }
-
- @Override
- public boolean isWaiting() {
- return false; //providers don't have inbound queues
- }
-
- @Override
- public void stopTask() {
- LOGGER.debug("Stopping Provider Task for {}", this.provider.getClass().getSimpleName());
- this.keepRunning.set(false);
- }
-
- @Override
- public void addInputQueue(BlockingQueue<StreamsDatum> inputQueue) {
- throw new UnsupportedOperationException(this.getClass().getName()+" does not support method - setInputQueue()");
- }
-
- @Override
- public void setStreamConfig(StreamsConfiguration config) {
- this.config = config;
- }
-
-
- @Override
- public void run() {
- try {
- this.provider.prepare(this.config); //TODO allow for configuration objects
- StreamsResultSet resultSet = null;
- //Negative values mean we want to run forever
- long maxZeros = timeout < 0 ? Long.MAX_VALUE : (timeout / sleepTime);
- if(this.counter == null) { //should never be null
- this.counter = new StreamsTaskCounter(this.provider.getClass().getName()+ UUID.randomUUID().toString(), getStreamIdentifier(), getStartedAt());
- }
- switch(this.type) {
- case PERPETUAL: {
- provider.startStream();
- this.started.set(true);
- while(this.isRunning()) {
- try {
- long startTime = System.currentTimeMillis();
- resultSet = provider.readCurrent();
- this.counter.addTime(System.currentTimeMillis() - startTime);
- if( resultSet.size() == 0 )
- zeros++;
- else {
- zeros = 0;
- }
- flushResults(resultSet);
- // the way this works needs to change...
- if(zeros > maxZeros)
- this.keepRunning.set(false);
- if(zeros > 0)
- Uninterruptibles.sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS);
- } catch (Exception e) {
- this.counter.incrementErrorCount();
- LOGGER.warn("Thread exception");
- this.keepRunning.set(false);
- }
- }
- Uninterruptibles.sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS);
- }
- break;
- case READ_CURRENT:
- resultSet = this.provider.readCurrent();
- this.started.set(true);
- break;
- case READ_NEW:
- resultSet = this.provider.readNew(this.sequence);
- this.started.set(true);
- break;
- case READ_RANGE:
- resultSet = this.provider.readRange(this.dateRange[START], this.dateRange[END]);
- this.started.set(true);
- break;
- default: throw new RuntimeException("Type has not been added to StreamsProviderTask.");
- }
- if( resultSet != null )
- flushResults(resultSet);
-
- } catch(Throwable e) {
- LOGGER.error("Caught Throwable in Provider {}", this.provider.getClass().getSimpleName(), e);
- } finally {
- Uninterruptibles.sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS);
- LOGGER.debug("Complete Provider Task execution for {}", this.provider.getClass().getSimpleName());
- this.provider.cleanUp();
- //Setting started to 'true' here will allow the isRunning() method to return false in the event of an exception
- //before started would normally be set to true n the run method.
- this.started.set(true);
- this.keepRunning.set(false);
- }
- }
-
- @Override
- public boolean isRunning() {
- //We want to make sure that we never return false if it is flushing, regardless of the state of the provider
- //or whether we have been told to shut down. If someone really wants us to shut down, they will interrupt the
- //thread and force us to shutdown. We also want to make sure we have had the opportunity to run before the
- //runtime kills us.
- return !this.started.get() || this.flushing.get() || (this.provider.isRunning() && this.keepRunning.get());
- }
-
- public void flushResults(StreamsResultSet resultSet) {
- Queue<StreamsDatum> queue = resultSet.getQueue();
- this.flushing.set(true);
- while(!queue.isEmpty()) {
- StreamsDatum datum = queue.poll();
- if(!this.keepRunning.get()) {
- break;
- }
- if(datum != null) {
- try {
- super.addToOutgoingQueue(datum);
- this.counter.incrementEmittedCount();
- statusCounter.incrementStatus(DatumStatus.SUCCESS);
- } catch( Exception e ) {
- this.counter.incrementErrorCount();
- statusCounter.incrementStatus(DatumStatus.FAIL);
- DatumUtils.addErrorToMetadata(datum, e, this.provider.getClass());
- }
- }
- }
- this.flushing.set(false);
- }
-
- @Override
- public void setStreamsTaskCounter(StreamsTaskCounter counter) {
- this.counter = counter;
- }
-
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
deleted file mode 100644
index 1b91e5c..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.tasks;
-
-import org.apache.streams.config.StreamsConfiguration;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.local.counters.StreamsTaskCounter;
-
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-/**
- * Interface for all task that will be used to execute instances of {@link org.apache.streams.core.StreamsOperation}
- * in local mode.
- */
-public interface StreamsTask extends Runnable{
-
- /**
- * Informs the task to stop. Tasks may or may not try to empty its inbound queue before halting.
- */
- public void stopTask();
-
- /**
- * Returns true if the task is waiting on more data to process
- * @return true, if waiting on more data to process
- */
- public boolean isWaiting();
- /**
- * Add an input {@link java.util.Queue} for this task.
- * @param inputQueue
- */
- public void addInputQueue(BlockingQueue<StreamsDatum> inputQueue);
-
- /**
- * Add an output {@link java.util.Queue} for this task.
- * @param outputQueue
- */
- public void addOutputQueue(BlockingQueue<StreamsDatum> outputQueue);
-
- /**
- * Set the configuration object that will shared and passed to all instances of StreamsTask.
- * @param config optional configuration information
- */
- public void setStreamConfig(StreamsConfiguration config);
-
- /**
- * Returns true when the task has not completed. Returns false otherwise
- * @return true when the task has not completed. Returns false otherwise
- */
- public boolean isRunning();
-
- /**
- * Returns the input queues that have been set for this task.
- * @return list of input queues
- */
- public List<BlockingQueue<StreamsDatum>> getInputQueues();
-
- /**
- * Returns the output queues that have been set for this task
- * @return list of output queues
- */
- public List<BlockingQueue<StreamsDatum>> getOutputQueues();
-
-
- public void setStreamsTaskCounter(StreamsTaskCounter counter);
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json b/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
deleted file mode 100644
index d5c7824..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id": "#",
- "type": "object",
- "javaType" : "org.apache.streams.local.LocalRuntimeConfiguration",
- "extends": {
- "$ref": "../../../../../streams-config/src/main/jsonschema/StreamsConfiguration.json"
- },
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "monitoring": {
- "type" : "object",
- "$ref": "../../../../../streams-monitoring/src/main/jsonschema/org/apache/streams/local/monitoring/MonitoringConfiguration.json"
- },
- "executorShutdownPauseMs": {
- "type": "integer",
- "description": "Duration of pause before shutting down"
- },
- "monitorShutdownPauseMs": {
- "type": "integer",
- "description": "Duration of pause before shutting down"
- },
- "executorShutdownWaitMs": {
- "type": "integer",
- "description": "Duration of wait for shut down"
- },
- "monitorShutdownWaitMs": {
- "type": "integer",
- "description": "Duration of wait for shut down"
- },
- "shutdownPauseMs": {
- "type": "integer",
- "description": "Duration of pause before shutting down"
- },
- "taskTimeoutMs": {
- "type": "integer",
- "description": "Max duration to allow tasks to terminate"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-runtimes/streams-runtime-local/src/main/resources/reference.conf b/streams-runtimes/streams-runtime-local/src/main/resources/reference.conf
deleted file mode 100644
index c9c5542..0000000
--- a/streams-runtimes/streams-runtime-local/src/main/resources/reference.conf
+++ /dev/null
@@ -1,6 +0,0 @@
-executorShutdownPauseMs = 3000
-monitorShutdownPauseMs = 3000
-executorShutdownWaitMs = 10000
-monitorShutdownWaitMs = 5000
-shutdownPauseMs = 1000
-taskTimeoutMs = 1000
diff --git a/streams-runtimes/streams-runtime-local/src/site/markdown/index.md b/streams-runtimes/streams-runtime-local/src/site/markdown/index.md
deleted file mode 100644
index 54e0c44..0000000
--- a/streams-runtimes/streams-runtime-local/src/site/markdown/index.md
+++ /dev/null
@@ -1,12 +0,0 @@
-streams-runtime-local
-==============
-
-streams-runtime-local can execute streams in single-process multi-threaded mode.
-
-## Configuration
-
-[LocalRuntimeConfiguration.json](LocalRuntimeConfiguration.json "LocalRuntimeConfiguration.json")
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
deleted file mode 100644
index 68e9dbc..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/LocalStreamBuilderTest.java
+++ /dev/null
@@ -1,407 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.builders;
-
-import org.apache.streams.core.StreamBuilder;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.local.counters.StreamsTaskCounter;
-import org.apache.streams.local.queues.ThroughputQueue;
-import org.apache.streams.local.test.processors.PassthroughDatumCounterProcessor;
-import org.apache.streams.local.test.processors.SlowProcessor;
-import org.apache.streams.local.test.providers.EmptyResultSetProvider;
-import org.apache.streams.local.test.providers.NumericMessageProvider;
-import org.apache.streams.local.test.writer.DatumCounterWriter;
-import org.apache.streams.local.test.writer.SystemOutWriter;
-import org.apache.streams.util.ComponentUtils;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import com.carrotsearch.randomizedtesting.annotations.Repeat;
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.joda.time.DateTime;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.lang.management.ManagementFactory;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.management.InstanceNotFoundException;
-import javax.management.MBeanRegistrationException;
-import javax.management.MBeanServer;
-import javax.management.MalformedObjectNameException;
-import javax.management.ObjectName;
-
-import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Basic Tests for the LocalStreamBuilder.
- *
- * Test are performed by redirecting system out and counting the number of lines that the SystemOutWriter prints
- * to System.out. The SystemOutWriter also prints one line when cleanUp() is called, so this is why it tests for
- * the numDatums +1.
- *
- *
- */
-public class LocalStreamBuilderTest extends RandomizedTest {
- private static final String MBEAN_ID = "test_id";
- private static final String STREAM_ID = "test_stream";
- private static long STREAM_START_TIME = (new DateTime()).getMillis();
-
- @After
- public void removeLocalMBeans() {
- try {
- ComponentUtils.removeAllMBeansOfDomain("org.apache.streams.local");
- } catch (Exception e) {
- //No op. proceed to next test
- }
- }
-
-
- public void removeRegisteredMBeans(String... ids) {
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- for(String id : ids) {
- try {
- mbs.unregisterMBean(new ObjectName(String.format(ThroughputQueue.NAME_TEMPLATE, id, STREAM_ID, STREAM_START_TIME)));
- } catch (MalformedObjectNameException|InstanceNotFoundException|MBeanRegistrationException e) {
- //No-op
- }
- try {
- mbs.unregisterMBean(new ObjectName((String.format(StreamsTaskCounter.NAME_TEMPLATE, id, STREAM_ID, STREAM_START_TIME))));
- } catch (MalformedObjectNameException|InstanceNotFoundException|MBeanRegistrationException e) {
- //No-op
- }
- }
- }
-
-
-
-
- @Test
- public void testStreamIdValidations() {
- StreamBuilder builder = new LocalStreamBuilder();
- builder.newReadCurrentStream("id", new NumericMessageProvider(1));
- Exception exp = null;
- try {
- builder.newReadCurrentStream("id", new NumericMessageProvider(1));
- } catch (RuntimeException e) {
- exp = e;
- }
- Assert.assertNotNull(exp);
- exp = null;
- builder.addStreamsProcessor("1", new PassthroughDatumCounterProcessor("1"), 1, "id");
- try {
- builder.addStreamsProcessor("2", new PassthroughDatumCounterProcessor("2"), 1, "id", "id2");
- } catch (RuntimeException e) {
- exp = e;
- }
- Assert.assertNotNull(exp);
- removeRegisteredMBeans("1", "2", "id");
- }
-
- @Test
- public void testBasicLinearStream1() {
- linearStreamNonParallel(1, 1);
- }
-
- @Test
- public void testBasicLinearStream2() {
- linearStreamNonParallel(1004, 1);
- }
-
- @Test
- public void testBasicLinearStream3() {
- linearStreamNonParallel(1, 10);
- }
-
- @Test
- @Repeat(iterations = 3)
- public void testBasicLinearStreamRandom() {
- int numDatums = randomIntBetween(1, 100000);
- int numProcessors = randomIntBetween(1, 10);
- linearStreamNonParallel(numDatums, numProcessors);
- }
-
- /**
- * Tests that all datums pass through each processor and that all datums reach the writer
- * @param numDatums
- * @param numProcessors
- */
- private void linearStreamNonParallel(int numDatums, int numProcessors) {
- String processorId = "proc";
- try {
- StreamBuilder builder = new LocalStreamBuilder(10);
- builder.newPerpetualStream("numeric_provider", new NumericMessageProvider(numDatums));
- String connectTo;
- for(int i=0; i < numProcessors; ++i) {
- if(i == 0) {
- connectTo = "numeric_provider";
- } else {
- connectTo = processorId+(i-1);
- }
- builder.addStreamsProcessor(processorId+i, new PassthroughDatumCounterProcessor(processorId+i), 1, connectTo);
- }
- Set output = Collections.newSetFromMap(new ConcurrentHashMap<>());
- builder.addStreamsPersistWriter("writer", new DatumCounterWriter("writer"), 1, processorId+(numProcessors-1));
- builder.start();
- for(int i=0; i < numProcessors; ++i) {
- Assert.assertEquals("Processor "+i+" did not receive all of the datums", numDatums, PassthroughDatumCounterProcessor.COUNTS.get(processorId+i).get());
- }
- for(int i=0; i < numDatums; ++i) {
- Assert.assertTrue("Expected writer to have received : "+i, DatumCounterWriter.RECEIVED.get("writer").contains(i));
- }
- } finally {
- for(int i=0; i < numProcessors; ++i) {
- removeRegisteredMBeans(processorId+i, processorId+i+"-"+PassthroughDatumCounterProcessor.class.getCanonicalName());
- }
- removeRegisteredMBeans("writer", "numeric_provider");
- }
- }
-
- @Test
- public void testParallelLinearStream1() {
- String processorId = "proc";
- int numProcessors = randomIntBetween(1, 10);
- int numDatums = randomIntBetween(1, 300000);
- try {
- StreamBuilder builder = new LocalStreamBuilder(50);
- builder.newPerpetualStream("numeric_provider", new NumericMessageProvider(numDatums));
- String connectTo;
- for(int i=0; i < numProcessors; ++i) {
- if(i == 0) {
- connectTo = "numeric_provider";
- } else {
- connectTo = processorId+(i-1);
- }
- int parallelHint = randomIntBetween(1,5);
- builder.addStreamsProcessor(processorId+i, new PassthroughDatumCounterProcessor(processorId+i), parallelHint, connectTo);
- }
- builder.addStreamsPersistWriter("writer", new DatumCounterWriter("writer"), 1, processorId+(numProcessors-1));
- builder.start();
- Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
- builder.stop();
- Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
- Assert.assertEquals(numDatums, DatumCounterWriter.RECEIVED.get("writer").size());
- for(int i=0; i < numDatums; ++i) {
- Assert.assertTrue("Expected Writer to receive datum : " + i, DatumCounterWriter.RECEIVED.get("writer").contains(i));
- }
- for(int i=0; i < numProcessors; ++i) {
- Assert.assertEquals(numDatums, PassthroughDatumCounterProcessor.COUNTS.get(processorId+i).get());
- }
-
- } finally {
- for(int i=0; i < numProcessors; ++i) {
- removeRegisteredMBeans(processorId+i);
- }
- removeRegisteredMBeans("writer", "numeric_provider");
- }
- }
-
- @Test
- public void testBasicMergeStream() {
- try {
- int numDatums1 = randomIntBetween(1, 300000);
- int numDatums2 = randomIntBetween(1, 300000);
- StreamsProcessor processor1 = new PassthroughDatumCounterProcessor("proc1");
- StreamsProcessor processor2 = new PassthroughDatumCounterProcessor("proc2");
- StreamBuilder builder = new LocalStreamBuilder();
- builder.newPerpetualStream("sp1", new NumericMessageProvider(numDatums1))
- .newPerpetualStream("sp2", new NumericMessageProvider(numDatums2))
- .addStreamsProcessor("proc1", processor1, 1, "sp1")
- .addStreamsProcessor("proc2", processor2, 1, "sp2")
- .addStreamsPersistWriter("writer1", new DatumCounterWriter("writer"), 1, "proc1", "proc2");
- builder.start();
- Assert.assertEquals(numDatums1, PassthroughDatumCounterProcessor.COUNTS.get("proc1").get());
- Assert.assertEquals(numDatums2, PassthroughDatumCounterProcessor.COUNTS.get("proc2").get());
- Assert.assertEquals(numDatums1+numDatums2, DatumCounterWriter.COUNTS.get("writer").get());
- } finally {
- String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
- String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
- removeRegisteredMBeans("proc1", "proc2", "writer1", "sp1", "sp2");
- }
- }
-
- @Test
- public void testBasicBranch() {
- try {
- int numDatums = randomIntBetween(1, 300000);
- StreamBuilder builder = new LocalStreamBuilder(50);
- builder.newPerpetualStream("prov1", new NumericMessageProvider(numDatums))
- .addStreamsProcessor("proc1", new PassthroughDatumCounterProcessor("proc1"), 1, "prov1")
- .addStreamsProcessor("proc2", new PassthroughDatumCounterProcessor("proc2"), 1, "prov1")
- .addStreamsPersistWriter("w1", new DatumCounterWriter("writer"), 1, "proc1", "proc2");
- builder.start();
- Assert.assertEquals(numDatums, PassthroughDatumCounterProcessor.COUNTS.get("proc1").get());
- Assert.assertEquals(numDatums, PassthroughDatumCounterProcessor.COUNTS.get("proc2").get());
- Assert.assertEquals(numDatums*2, DatumCounterWriter.COUNTS.get("writer").get());
- } finally {
- String provClass = "-"+NumericMessageProvider.class.getCanonicalName();
- String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
- String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
- removeRegisteredMBeans("prov1", "proc1", "proc2", "w1");
- }
- }
-
- @Test
- public void testSlowProcessorBranch() {
- try {
- int numDatums = 30;
- int timeout = 2000;
- Map<String, Object> config = new HashMap<>();
- config.put(LocalStreamBuilder.TIMEOUT_KEY, timeout);
- StreamBuilder builder = new LocalStreamBuilder(config);
- builder.newPerpetualStream("prov1", new NumericMessageProvider(numDatums))
- .addStreamsProcessor("proc1", new SlowProcessor(), 1, "prov1")
- .addStreamsPersistWriter("w1", new DatumCounterWriter("writer"), 1, "proc1");
- builder.start();
- Assert.assertEquals(numDatums, DatumCounterWriter.COUNTS.get("writer").get());
- } finally {
- String provClass = "-"+NumericMessageProvider.class.getCanonicalName();
- String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
- String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
- removeRegisteredMBeans("prov1", "proc1", "w1");
- }
- }
-
- @Test
- public void testConfiguredProviderTimeout() {
- try {
- Map<String, Object> config = new HashMap<>();
- int timeout = 10000;
- config.put(LocalStreamBuilder.TIMEOUT_KEY, timeout);
- long start = System.currentTimeMillis();
- StreamBuilder builder = new LocalStreamBuilder(-1, config);
- builder.newPerpetualStream("prov1", new EmptyResultSetProvider())
- .addStreamsProcessor("proc1", new PassthroughDatumCounterProcessor("proc1"), 1, "prov1")
- .addStreamsProcessor("proc2", new PassthroughDatumCounterProcessor("proc2"), 1, "proc1")
- .addStreamsPersistWriter("w1", new SystemOutWriter(), 1, "proc1");
- builder.start();
- long end = System.currentTimeMillis();
- //We care mostly that it doesn't terminate too early. With thread shutdowns, etc, the actual time is indeterminate. Just make sure there is an upper bound
- Assert.assertThat((int) (end - start), is(allOf(greaterThanOrEqualTo(timeout), lessThanOrEqualTo(4 * timeout))));
- } finally {
- String provClass = "-"+NumericMessageProvider.class.getCanonicalName();
- String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
- String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
- removeRegisteredMBeans("prov1", "proc1", "proc2", "w1");
- }
- }
-
- @Ignore
- @Test
- public void ensureShutdownWithBlockedQueue() throws InterruptedException {
- try {
- ExecutorService service = Executors.newSingleThreadExecutor();
- int before = Thread.activeCount();
- final StreamBuilder builder = new LocalStreamBuilder();
- builder.newPerpetualStream("prov1", new NumericMessageProvider(30))
- .addStreamsProcessor("proc1", new SlowProcessor(), 1, "prov1")
- .addStreamsPersistWriter("w1", new SystemOutWriter(), 1, "proc1");
- service.submit(builder::start);
- //Let streams spin up threads and start to process
- Thread.sleep(500);
- builder.stop();
- service.shutdownNow();
- service.awaitTermination(30000, TimeUnit.MILLISECONDS);
- Assert.assertThat(Thread.activeCount(), is(equalTo(before)));
- } finally {
- String provClass = "-"+NumericMessageProvider.class.getCanonicalName();
- String procClass = "-"+PassthroughDatumCounterProcessor.class.getCanonicalName();
- String writerClass = "-"+DatumCounterWriter.class.getCanonicalName();
- removeRegisteredMBeans("prov1", "proc1", "w1");
- }
- }
-
- @Before
- private void clearCounters() {
- PassthroughDatumCounterProcessor.COUNTS.clear();
- PassthroughDatumCounterProcessor.CLAIMED_ID.clear();
- PassthroughDatumCounterProcessor.SEEN_DATA.clear();
- DatumCounterWriter.COUNTS.clear();
- DatumCounterWriter.CLAIMED_ID.clear();
- DatumCounterWriter.SEEN_DATA.clear();
- DatumCounterWriter.RECEIVED.clear();
- }
-
-
- /**
- * Creates {@link org.apache.streams.core.StreamsProcessor} that passes any StreamsDatum it gets as an
- * input and counts the number of items it processes.
- * @param counter
- * @return
- */
- private StreamsProcessor createPassThroughProcessor(final AtomicInteger counter) {
- StreamsProcessor processor = mock(StreamsProcessor.class);
- when(processor.process(any(StreamsDatum.class))).thenAnswer(new Answer<List<StreamsDatum>>() {
- @Override
- public List<StreamsDatum> answer(InvocationOnMock invocationOnMock) throws Throwable {
- List<StreamsDatum> datum = new LinkedList<>();
- if(counter != null) {
- counter.incrementAndGet();
- }
- datum.add((StreamsDatum) invocationOnMock.getArguments()[0] );
- return datum;
- }
- });
- return processor;
- }
-
- private StreamsPersistWriter createSetCollectingWriter(final Set collector) {
- return createSetCollectingWriter(collector, null);
- }
-
- /**
- * Creates a StreamsPersistWriter that adds every datums document to a set
- * @param collector
- * @return
- */
- private StreamsPersistWriter createSetCollectingWriter(final Set collector, final AtomicInteger counter) {
- StreamsPersistWriter writer = mock(StreamsPersistWriter.class);
- doAnswer(invocationOnMock -> {
- if(counter != null) {
- counter.incrementAndGet();
- }
- collector.add(((StreamsDatum)invocationOnMock.getArguments()[0]).getDocument());
- return null;
- }).when(writer).write(any(StreamsDatum.class));
- return writer;
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/ToyLocalBuilderExample.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/ToyLocalBuilderExample.java
deleted file mode 100644
index 2c61093..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/builders/ToyLocalBuilderExample.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.builders;
-
-import org.apache.streams.core.StreamBuilder;
-import org.apache.streams.local.test.processors.DoNothingProcessor;
-import org.apache.streams.local.test.providers.NumericMessageProvider;
-import org.apache.streams.local.test.writer.DoNothingWriter;
-
-
-
-/**
- * Created by rebanks on 2/20/14.
- */
-public class ToyLocalBuilderExample {
-
- /**
- * A simple example of how to run a stream in local mode.
- * @param args
- */
- public static void main(String[] args) {
- StreamBuilder builder = new LocalStreamBuilder();
- builder.newReadCurrentStream("prov", new NumericMessageProvider(1000000))
- .addStreamsProcessor("proc", new DoNothingProcessor(), 100, "prov")
- .addStreamsPersistWriter("writer", new DoNothingWriter(), 3, "proc");
- builder.start();
- }
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/counters/DatumStatusCounterTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/counters/DatumStatusCounterTest.java
deleted file mode 100644
index 0d4b1be..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/counters/DatumStatusCounterTest.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.streams.local.counters;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import com.carrotsearch.randomizedtesting.annotations.Repeat;
-import org.joda.time.DateTime;
-import org.junit.After;
-import org.junit.Test;
-
-import java.lang.management.ManagementFactory;
-import javax.management.InstanceNotFoundException;
-import javax.management.ObjectName;
-
-import org.junit.Assert;
-
-
-/**
- *
- */
-public class DatumStatusCounterTest extends RandomizedTest {
-
- private static final String MBEAN_ID = "test_id";
- private static final String STREAM_ID = "test_stream";
- private static long STREAM_START_TIME = (new DateTime()).getMillis();
-
-
- /**
- * Remove registered mbeans from previous tests
- * @throws Exception
- */
- @After
- public void unregisterMXBean() throws Exception {
- try {
- ManagementFactory.getPlatformMBeanServer().unregisterMBean(new ObjectName(String.format(DatumStatusCounter.NAME_TEMPLATE, MBEAN_ID, STREAM_ID, STREAM_START_TIME)));
- } catch (InstanceNotFoundException ife) {
- //No-op
- }
- }
-
- /**
- * Test Constructor can register the counter as an mxbean with throwing an exception.
- */
- @Test
- public void testConstructor() {
- try {
- new DatumStatusCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- } catch (Throwable t) {
- Assert.fail("Constructor Threw Exception : "+t.getMessage());
- }
- }
-
- /**
- * Test that you can increment passes and it returns the correct count
- * @throws Exception
- */
- @Test
- @Repeat(iterations = 3)
- public void testPassed() throws Exception {
- DatumStatusCounter counter = new DatumStatusCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- int numIncrements = randomIntBetween(1, 100000);
- for(int i=0; i < numIncrements; ++i) {
- counter.incrementPassedCount();
- }
- Assert.assertEquals(numIncrements, counter.getNumPassed());
-
- unregisterMXBean();
-
- counter = new DatumStatusCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- numIncrements = randomIntBetween(1, 100000);
- long total = 0;
- for(int i=0; i < numIncrements; ++i) {
- long delta = randomIntBetween(1, 100);
- total += delta;
- counter.incrementPassedCount(delta);
- }
- Assert.assertEquals(total, counter.getNumPassed());
- }
-
- /**
- * Test that you can increment failed and it returns the correct count
- * @throws Exception
- */
- @Test
- @Repeat(iterations = 3)
- public void testFailed() throws Exception {
- DatumStatusCounter counter = new DatumStatusCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- int numIncrements = randomIntBetween(1, 100000);
- for(int i=0; i < numIncrements; ++i) {
- counter.incrementFailedCount();
- }
- Assert.assertEquals(numIncrements, counter.getNumFailed());
-
- unregisterMXBean();
-
- counter = new DatumStatusCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- numIncrements = randomIntBetween(1, 100000);
- long total = 0;
- for(int i=0; i < numIncrements; ++i) {
- long delta = randomIntBetween(1, 100);
- total += delta;
- counter.incrementFailedCount(delta);
- }
- Assert.assertEquals(total, counter.getNumFailed());
- }
-
-
- /**
- * Test failure rate returns expected values
- */
- @Test
- @Repeat(iterations = 3)
- public void testFailureRate() {
- DatumStatusCounter counter = new DatumStatusCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- Assert.assertEquals(0.0, counter.getFailRate(), 0);
- int failures = randomIntBetween(0, 100000);
- int passes = randomIntBetween(0, 100000);
- counter.incrementPassedCount(passes);
- counter.incrementFailedCount(failures);
- Assert.assertEquals((double)failures / (double)(passes + failures), counter.getFailRate(), 0);
- }
-}
\ No newline at end of file
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/counters/StreamsTaskCounterTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/counters/StreamsTaskCounterTest.java
deleted file mode 100644
index 30ed057..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/counters/StreamsTaskCounterTest.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.streams.local.counters;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import com.carrotsearch.randomizedtesting.annotations.Repeat;
-import org.joda.time.DateTime;
-import org.junit.After;
-import org.junit.Test;
-
-import java.lang.management.ManagementFactory;
-import javax.management.InstanceNotFoundException;
-import javax.management.ObjectName;
-
-import org.junit.Assert;
-
-
-/**
- * Unit tests for {@link org.apache.streams.local.counters.StreamsTaskCounter}
- */
-public class StreamsTaskCounterTest extends RandomizedTest {
-
- private static final String MBEAN_ID = "test_id";
- private static final String STREAM_ID = "test_stream";
- private static long STREAM_START_TIME = (new DateTime()).getMillis();
-
- /**
- * Remove registered mbeans from previous tests
- * @throws Exception
- */
- @After
- public void unregisterMXBean() throws Exception {
- try {
- ManagementFactory.getPlatformMBeanServer().unregisterMBean(new ObjectName(String.format(StreamsTaskCounter.NAME_TEMPLATE, MBEAN_ID, STREAM_ID, STREAM_START_TIME)));
- } catch (InstanceNotFoundException ife) {
- //No-op
- }
- }
-
- /**
- * Test constructor does not throw errors
- */
- @Test
- public void testConstructor() {
- try {
- new StreamsTaskCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- } catch (Throwable t) {
- Assert.fail("Constructor threw error : "+t.getMessage());
- }
- }
-
- /**
- * Test emitted increments correctly and returns expected value
- * @throws Exception
- */
- @Test
- @Repeat(iterations = 3)
- public void testEmitted() throws Exception {
- StreamsTaskCounter counter = new StreamsTaskCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- int numIncrements = randomIntBetween(1, 100000);
- for(int i=0; i < numIncrements; ++i) {
- counter.incrementEmittedCount();
- }
- Assert.assertEquals(numIncrements, counter.getNumEmitted());
-
- unregisterMXBean();
-
- counter = new StreamsTaskCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- numIncrements = randomIntBetween(1, 100000);
- long total = 0;
- for(int i=0; i < numIncrements; ++i) {
- long delta = randomIntBetween(1, 100);
- total += delta;
- counter.incrementEmittedCount(delta);
- }
- Assert.assertEquals(total, counter.getNumEmitted());
- }
-
- /**
- * Test received increments correctly and returns expected value
- * @throws Exception
- */
- @Test
- @Repeat(iterations = 3)
- public void testReceived() throws Exception {
- StreamsTaskCounter counter = new StreamsTaskCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- int numIncrements = randomIntBetween(1, 100000);
- for(int i=0; i < numIncrements; ++i) {
- counter.incrementReceivedCount();
- }
- Assert.assertEquals(numIncrements, counter.getNumReceived());
-
- unregisterMXBean();
-
- counter = new StreamsTaskCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- numIncrements = randomIntBetween(1, 100000);
- long total = 0;
- for(int i=0; i < numIncrements; ++i) {
- long delta = randomIntBetween(1, 100);
- total += delta;
- counter.incrementReceivedCount(delta);
- }
- Assert.assertEquals(total, counter.getNumReceived());
- }
-
- /**
- * Test errors increments correctly and returns expected value
- * @throws Exception
- */
- @Test
- @Repeat(iterations = 3)
- public void testError() throws Exception {
- StreamsTaskCounter counter = new StreamsTaskCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- int numIncrements = randomIntBetween(1, 100000);
- for(int i=0; i < numIncrements; ++i) {
- counter.incrementErrorCount();
- }
- Assert.assertEquals(numIncrements, counter.getNumUnhandledErrors());
-
- unregisterMXBean();
-
- counter = new StreamsTaskCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- numIncrements = randomIntBetween(1, 100000);
- long total = 0;
- for(int i=0; i < numIncrements; ++i) {
- long delta = randomIntBetween(1, 100);
- total += delta;
- counter.incrementErrorCount(delta);
- }
- Assert.assertEquals(total, counter.getNumUnhandledErrors());
- }
-
- /**
- * Test error rate returns expected value
- * @throws Exception
- */
- @Test
- @Repeat(iterations = 3)
- public void testErrorRate() throws Exception {
- StreamsTaskCounter counter = new StreamsTaskCounter(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- Assert.assertEquals(0.0, counter.getErrorRate(), 0);
- int failures = randomIntBetween(0, 100000);
- int received = randomIntBetween(0, 100000);
- counter.incrementReceivedCount(received);
- counter.incrementErrorCount(failures);
- Assert.assertEquals((double)failures / (double)(received), counter.getErrorRate(), 0);
- }
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/executors/ShutdownStreamOnUnhandledThrowableThreadPoolExecutorTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/executors/ShutdownStreamOnUnhandledThrowableThreadPoolExecutorTest.java
deleted file mode 100644
index 0048153..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/executors/ShutdownStreamOnUnhandledThrowableThreadPoolExecutorTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.executors;
-
-import org.apache.streams.local.builders.LocalStreamBuilder;
-import org.apache.streams.util.ComponentUtils;
-
-import org.junit.After;
-import org.junit.Test;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-
-/**
- *
- */
-public class ShutdownStreamOnUnhandledThrowableThreadPoolExecutorTest {
-
-
- @After
- public void removeLocalMBeans() {
- try {
- ComponentUtils.removeAllMBeansOfDomain("org.apache.streams.local");
- } catch (Exception e) {
- //No op. proceed to next test
- }
- }
-
- @Test
- public void testShutDownOnException() {
- LocalStreamBuilder sb = mock(LocalStreamBuilder.class);
- final AtomicBoolean isShutdown = new AtomicBoolean(false);
- doAnswer(invocationOnMock -> {
- isShutdown.set(true);
- return null;
- }).when(sb).stop();
-
- final CountDownLatch latch = new CountDownLatch(1);
-
- Runnable runnable = () -> {
- latch.countDown();
- throw new RuntimeException("Testing Throwable Handling!");
- };
-
- ExecutorService executor = new ShutdownStreamOnUnhandleThrowableThreadPoolExecutor(1, sb);
- executor.execute(runnable);
- try {
- latch.await();
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- executor.shutdownNow();
- try {
- executor.awaitTermination(1, TimeUnit.SECONDS);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- assertTrue("Expected StreamBuilder shutdown to be called", isShutdown.get());
- }
-
-
- @Test
- public void testNormalExecution() {
- LocalStreamBuilder sb = mock(LocalStreamBuilder.class);
- final AtomicBoolean isShutdown = new AtomicBoolean(false);
- doAnswer(invocationOnMock -> {
- isShutdown.set(true);
- return null;
- }).when(sb).stop();
-
- final CountDownLatch latch = new CountDownLatch(1);
-
- Runnable runnable = latch::countDown;
-
- ExecutorService executor = new ShutdownStreamOnUnhandleThrowableThreadPoolExecutor(1, sb);
- executor.execute(runnable);
- try {
- latch.await();
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- executor.shutdownNow();
- try {
- executor.awaitTermination(1, TimeUnit.SECONDS);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- assertFalse("Expected StreamBuilder shutdown to be called", isShutdown.get());
- }
-
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/queues/ThroughputQueueMultiThreadTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/queues/ThroughputQueueMultiThreadTest.java
deleted file mode 100644
index 1aa0c2b..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/queues/ThroughputQueueMultiThreadTest.java
+++ /dev/null
@@ -1,325 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.streams.local.queues;
-
-import org.apache.streams.util.ComponentUtils;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import com.carrotsearch.randomizedtesting.annotations.Repeat;
-import org.joda.time.DateTime;
-import org.junit.After;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.management.ManagementFactory;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import javax.management.InstanceNotFoundException;
-import javax.management.ObjectName;
-
-import org.junit.Assert;
-
-
-
-
-/**
- * MultiThread unit tests for {@link org.apache.streams.local.queues.ThroughputQueue}
- */
-public class ThroughputQueueMultiThreadTest extends RandomizedTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ThroughputQueueMultiThreadTest.class);
- private static final String MBEAN_ID = "testQueue";
- private static final String STREAM_ID = "test_stream";
- private static long STREAM_START_TIME = (new DateTime()).getMillis();
-
- /**
- * Remove registered mbeans from previous tests
- * @throws Exception
- */
- @After
- public void unregisterMXBean() throws Exception {
- try {
- ManagementFactory.getPlatformMBeanServer().unregisterMBean(new ObjectName(String.format(ThroughputQueue.NAME_TEMPLATE, MBEAN_ID, STREAM_ID, STREAM_START_TIME)));
- } catch (InstanceNotFoundException ife) {
- //No-op
- }
- }
-
- @After
- public void removeLocalMBeans() {
- try {
- ComponentUtils.removeAllMBeansOfDomain("org.apache.streams.local");
- } catch (Exception e) {
- //No op. proceed to next test
- }
- }
-
-
- /**
- * Test that queue will block on puts when the queue is full
- * @throws InterruptedException
- */
- @Test
- public void testBlockOnFullQueue() throws InterruptedException {
- int queueSize = randomIntBetween(1, 3000);
- ExecutorService executor = Executors.newSingleThreadExecutor();
- CountDownLatch full = new CountDownLatch(1);
- CountDownLatch finished = new CountDownLatch(1);
- ThroughputQueue queue = new ThroughputQueue(queueSize);
- BlocksOnFullQueue testThread = new BlocksOnFullQueue(full, finished, queue, queueSize);
- executor.submit(testThread);
- full.await();
- Assert.assertEquals(queueSize, queue.size());
- Assert.assertEquals(queueSize, queue.getCurrentSize());
- Assert.assertFalse(testThread.isComplete()); //test that it is blocked
- safeSleep(1000);
- Assert.assertFalse(testThread.isComplete()); //still blocked
- queue.take();
- finished.await();
- Assert.assertEquals(queueSize, queue.size());
- Assert.assertEquals(queueSize, queue.getCurrentSize());
- Assert.assertTrue(testThread.isComplete());
- executor.shutdownNow();
- executor.awaitTermination(500, TimeUnit.MILLISECONDS);
- }
-
- /**
- * Test that queue will block on Take when queue is empty
- * @throws InterruptedException
- */
- @Test
- public void testBlockOnEmptyQueue() throws InterruptedException {
- int queueSize = randomIntBetween(1, 3000);
- ExecutorService executor = Executors.newSingleThreadExecutor();
- CountDownLatch empty = new CountDownLatch(1);
- CountDownLatch finished = new CountDownLatch(1);
- ThroughputQueue queue = new ThroughputQueue();
- BlocksOnEmptyQueue testThread = new BlocksOnEmptyQueue(empty, finished, queueSize, queue);
- for(int i=0; i < queueSize; ++i) {
- queue.put(i);
- }
- executor.submit(testThread);
- empty.await();
- Assert.assertEquals(0, queue.size());
- Assert.assertEquals(0, queue.getCurrentSize());
- Assert.assertFalse(testThread.isComplete());
- safeSleep(1000);
- Assert.assertFalse(testThread.isComplete());
- queue.put(1);
- finished.await();
- Assert.assertEquals(0, queue.size());
- Assert.assertEquals(0, queue.getCurrentSize());
- Assert.assertTrue(testThread.isComplete());
- executor.shutdownNow();
- executor.awaitTermination(500, TimeUnit.MILLISECONDS);
- }
-
-
- /**
- * Test multiple threads putting and taking from the queue while
- * this thread repeatedly calls the MXBean measurement methods.
- * Should hammer the queue with request from multiple threads
- * of all request types. Purpose is to expose current modification exceptions
- * and/or dead locks.
- */
- @Test
- @Repeat(iterations = 3)
- public void testMultiThreadAccessAndInteruptResponse() throws Exception {
- int putTakeThreadCount = randomIntBetween(1, 10);
- int dataCount = randomIntBetween(1, 2000000);
- int pollCount = randomIntBetween(1, 2000000);
- int maxSize = randomIntBetween(1, 1000);
- CountDownLatch finished = new CountDownLatch(putTakeThreadCount);
- ThroughputQueue queue = new ThroughputQueue(maxSize, MBEAN_ID);
- ExecutorService executor = Executors.newFixedThreadPool(putTakeThreadCount * 2);
- for(int i=0; i < putTakeThreadCount; ++i) {
- executor.submit(new PutData(finished, queue, dataCount));
- executor.submit(new TakeData(queue));
- }
- for(int i=0; i < pollCount; ++i) {
- queue.getAvgWait();
- queue.getAdded();
- queue.getCurrentSize();
- queue.getMaxWait();
- queue.getRemoved();
- queue.getThroughput();
- }
- finished.await();
- while(!queue.isEmpty()) {
- LOGGER.info("Waiting for queue to be emptied...");
- safeSleep(500);
- }
- long totalData = ((long) dataCount) * putTakeThreadCount;
- Assert.assertEquals(totalData, queue.getAdded());
- Assert.assertEquals(totalData, queue.getRemoved());
- executor.shutdown();
- executor.awaitTermination(1000, TimeUnit.MILLISECONDS); //shutdown puts
- executor.shutdownNow();
- executor.awaitTermination(1000, TimeUnit.MILLISECONDS); //shutdown takes
- //Randomized should not report thread leak
- }
-
-
-
- private void safeSleep(long sleep) {
- try {
- Thread.sleep(sleep);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
-
-
-
-
- /**
- * Helper runnable for test {@link ThroughputQueueMultiThreadTest#testBlockOnFullQueue()}
- */
- private class BlocksOnFullQueue implements Runnable {
-
- private CountDownLatch full;
- volatile private boolean complete;
- private int queueSize;
- private CountDownLatch finished;
- private BlockingQueue queue;
-
- public BlocksOnFullQueue(CountDownLatch latch, CountDownLatch finished, BlockingQueue queue, int queueSize) {
- this.full = latch;
- this.complete = false;
- this.queueSize = queueSize;
- this.finished = finished;
- this.queue = queue;
- }
-
- @Override
- public void run() {
- try {
- for (int i = 0; i < this.queueSize; ++i) {
- this.queue.put(i);
- }
- this.full.countDown();
- this.queue.put(0);
- this.complete = true;
- this.finished.countDown();
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
-
- public boolean isComplete() {
- return this.complete;
- }
- }
-
-
- /**
- * Helper runnable class for test {@link ThroughputQueueMultiThreadTest#testBlockOnEmptyQueue()}
- */
- private class BlocksOnEmptyQueue implements Runnable {
-
- private CountDownLatch full;
- volatile private boolean complete;
- private int queueSize;
- private CountDownLatch finished;
- private BlockingQueue queue;
-
- public BlocksOnEmptyQueue(CountDownLatch full, CountDownLatch finished, int queueSize, BlockingQueue queue) {
- this.full = full;
- this.finished = finished;
- this.queueSize = queueSize;
- this.queue = queue;
- this.complete = false;
- }
-
-
- @Override
- public void run() {
- try {
- for(int i=0; i < this.queueSize; ++i) {
- this.queue.take();
- }
- this.full.countDown();
- this.queue.take();
- this.complete = true;
- this.finished.countDown();
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
-
- public boolean isComplete() {
- return this.complete;
- }
- }
-
-
- private class PutData implements Runnable {
-
- private BlockingQueue queue;
- private int dataCount;
- private CountDownLatch finished;
-
- public PutData(CountDownLatch finished, BlockingQueue queue, int dataCount) {
- this.queue = queue;
- this.dataCount = dataCount;
- this.finished = finished;
- }
-
-
- @Override
- public void run() {
- try {
- for(int i=0; i < this.dataCount; ++i) {
- this.queue.put(i);
- }
- } catch (InterruptedException ie) {
- LOGGER.error("PUT DATA interupted !");
- Thread.currentThread().interrupt();
- }
- this.finished.countDown();
- }
- }
-
-
- private class TakeData implements Runnable {
-
- private BlockingQueue queue;
-
- public TakeData(BlockingQueue queue) {
- this.queue = queue;
- }
-
-
- @Override
- public void run() {
- try {
- while(true) {
- this.queue.take();
- }
- } catch (InterruptedException ie) {
- LOGGER.error("PUT DATA interupted !");
- Thread.currentThread().interrupt();
- }
- }
- }
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/queues/ThroughputQueueSingleThreadTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/queues/ThroughputQueueSingleThreadTest.java
deleted file mode 100644
index fafc913..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/queues/ThroughputQueueSingleThreadTest.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.streams.local.queues;
-
-import org.apache.streams.util.ComponentUtils;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import com.carrotsearch.randomizedtesting.annotations.Repeat;
-import org.joda.time.DateTime;
-import org.junit.After;
-import org.junit.Test;
-
-import java.lang.management.ManagementFactory;
-import javax.management.MBeanServer;
-import javax.management.ObjectInstance;
-import javax.management.ObjectName;
-
-import org.junit.Assert;
-
-
-
-/**
- * Single thread unit tests for {@link org.apache.streams.local.queues.ThroughputQueue}
- */
-public class ThroughputQueueSingleThreadTest extends RandomizedTest {
- private static final String MBEAN_ID = "test_id";
- private static final String STREAM_ID = "test_stream";
- private static long STREAM_START_TIME = (new DateTime()).getMillis();
-
- @After
- public void removeLocalMBeans() {
- try {
- ComponentUtils.removeAllMBeansOfDomain("org.apache.streams.local");
- } catch (Exception e) {
- //No op. proceed to next test
- }
- }
-
- /**
- * Test that take and put queue and dequeue data as expected and all
- * measurements form the queue are returning data.
- * @throws Exception
- */
- @Test
- @Repeat(iterations = 3)
- public void testTakeAndPut() throws Exception {
- ThroughputQueue<Integer> queue = new ThroughputQueue<>();
- int putCount = randomIntBetween(1, 1000);
- for(int i=0; i < putCount; ++i) {
- queue.put(i);
- Assert.assertEquals(i+1, queue.size());
- Assert.assertEquals(queue.size(), queue.getCurrentSize());
- }
- safeSleep(100); //ensure measurable wait time
- int takeCount = randomIntBetween(1, putCount);
- for(int i=0; i < takeCount; ++i) {
- Integer element = queue.take();
- Assert.assertNotNull(element);
- Assert.assertEquals(i, element.intValue());
- Assert.assertEquals(putCount - (1+i), queue.size());
- Assert.assertEquals(queue.size(), queue.getCurrentSize());
- }
- Assert.assertEquals(putCount-takeCount, queue.size());
- Assert.assertEquals(queue.size(), queue.getCurrentSize());
- Assert.assertTrue(0.0 < queue.getMaxWait());
- Assert.assertTrue(0.0 < queue.getAvgWait());
- Assert.assertTrue(0.0 < queue.getThroughput());
- Assert.assertEquals(putCount, queue.getAdded());
- Assert.assertEquals(takeCount, queue.getRemoved());
- }
-
- /**
- * Test that add and remove queue and dequeue data as expected
- * and all measurements from the queue are returning data
- */
- @Test
- @Repeat(iterations = 3)
- public void testAddAndRemove() {
- ThroughputQueue<Integer> queue = new ThroughputQueue<>();
- int putCount = randomIntBetween(1, 1000);
- for(int i=0; i < putCount; ++i) {
- queue.add(i);
- Assert.assertEquals(i+1, queue.size());
- Assert.assertEquals(queue.size(), queue.getCurrentSize());
- }
- safeSleep(100); //ensure measurable wait time
- int takeCount = randomIntBetween(1, putCount);
- for(int i=0; i < takeCount; ++i) {
- Integer element = queue.remove();
- Assert.assertNotNull(element);
- Assert.assertEquals(i, element.intValue());
- Assert.assertEquals(putCount - (1+i), queue.size());
- Assert.assertEquals(queue.size(), queue.getCurrentSize());
- }
- Assert.assertEquals(putCount-takeCount, queue.size());
- Assert.assertEquals(queue.size(), queue.getCurrentSize());
- Assert.assertTrue(0.0 < queue.getMaxWait());
- Assert.assertTrue(0.0 < queue.getAvgWait());
- Assert.assertTrue(0.0 < queue.getThroughput());
- Assert.assertEquals(putCount, queue.getAdded());
- Assert.assertEquals(takeCount, queue.getRemoved());
- }
-
- /**
- * Test that offer and poll queue and dequeue data as expected
- * and all measurements from the queue are returning data
- */
- @Test
- @Repeat(iterations = 3)
- public void testOfferAndPoll() {
- ThroughputQueue<Integer> queue = new ThroughputQueue<>();
- int putCount = randomIntBetween(1, 1000);
- for(int i=0; i < putCount; ++i) {
- queue.offer(i);
- Assert.assertEquals(i+1, queue.size());
- Assert.assertEquals(queue.size(), queue.getCurrentSize());
- }
- safeSleep(100); //ensure measurable wait time
- int takeCount = randomIntBetween(1, putCount);
- for(int i=0; i < takeCount; ++i) {
- Integer element = queue.poll();
- Assert.assertNotNull(element);
- Assert.assertEquals(i, element.intValue());
- Assert.assertEquals(putCount - (1+i), queue.size());
- Assert.assertEquals(queue.size(), queue.getCurrentSize());
- }
- Assert.assertEquals(putCount-takeCount, queue.size());
- Assert.assertEquals(queue.size(), queue.getCurrentSize());
- Assert.assertTrue(0.0 < queue.getMaxWait());
- Assert.assertTrue(0.0 < queue.getAvgWait());
- Assert.assertTrue(0.0 < queue.getThroughput());
- Assert.assertEquals(putCount, queue.getAdded());
- Assert.assertEquals(takeCount, queue.getRemoved());
- }
-
-
-
- /**
- * Test that max wait and avg wait return expected values
- * @throws Exception
- */
- @Test
- public void testWait() throws Exception {
- ThroughputQueue queue = new ThroughputQueue();
- int wait = 1000;
-
- for(int i=0; i < 3; ++i) {
- queue.put(1);
- safeSleep(wait);
- queue.take();
- Assert.assertTrue(queue.getMaxWait() >= wait && queue.getMaxWait() <= (wait * 2));//can't calculate exactly, making sure its close.
- Assert.assertTrue(queue.getAvgWait() >= wait && queue.getAvgWait() <= (wait * 2));
- }
- queue.put(1);
- queue.take();
- Assert.assertTrue(queue.getMaxWait() >= wait && queue.getMaxWait() <= (wait * 2));//can't calculate exactly, making sure its close.
- Assert.assertTrue(queue.getAvgWait() <= 5000 );
- Assert.assertTrue(queue.getAvgWait() >= 500);
- }
-
- /**
- * Test that throughput returns expected values.
- * @throws Exception
- */
- @Test
- public void testThroughput() throws Exception {
- ThroughputQueue queue = new ThroughputQueue();
- int wait = 100;
- for(int i=0; i < 10; ++i) {
- queue.put(1);
- safeSleep(wait);
- queue.take();
- }
- double throughput = queue.getThroughput();
- Assert.assertTrue(throughput <= 15 ); //can't calculate exactly, making sure its close.
- Assert.assertTrue(throughput >= 5);
-
- queue = new ThroughputQueue();
- wait = 1000;
- for(int i=0; i < 10; ++i) {
- queue.put(1);
- }
- for(int i=0; i < 10; ++i) {
- queue.take();
- }
- safeSleep(wait);
- throughput = queue.getThroughput();
- Assert.assertTrue(throughput <= 15 ); //can't calculate exactly, making sure its close.
- Assert.assertTrue(throughput >= 5);
- }
-
-
- /**
- * Test that the mbean registers
- */
- @Test
- public void testMBeanRegistration() {
- try {
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- Integer beanCount = mbs.getMBeanCount();
- ThroughputQueue queue = new ThroughputQueue(MBEAN_ID, STREAM_ID, STREAM_START_TIME);
- Assert.assertEquals("Expected bean to be registered", new Integer(beanCount+1), mbs.getMBeanCount());
- ObjectInstance mBean = mbs.getObjectInstance(new ObjectName(String.format(ThroughputQueue.NAME_TEMPLATE, MBEAN_ID, STREAM_ID, STREAM_START_TIME)));
- Assert.assertNotNull(mBean);
- } catch (Exception e) {
- Assert.fail("Assert.failed to register MXBean : "+e.getMessage());
- }
- }
-
- /**
- * Test that mulitple mbeans of the same type with a different name can be registered
- */
- @Test
- public void testMultipleMBeanRegistrations() {
- try {
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- Integer beanCount = mbs.getMBeanCount();
- int numReg = randomIntBetween(2, 100);
- for(int i=0; i < numReg; ++i) {
- ThroughputQueue queue = new ThroughputQueue(MBEAN_ID + "" + i, STREAM_ID, STREAM_START_TIME);
- Assert.assertEquals("Expected bean to be registered", new Integer(beanCount + (i+1)), mbs.getMBeanCount());
- ObjectInstance mBean = mbs.getObjectInstance(new ObjectName(String.format(ThroughputQueue.NAME_TEMPLATE, MBEAN_ID + "" + i, STREAM_ID, STREAM_START_TIME)));
- Assert.assertNotNull(mBean);
- }
- } catch (Exception e) {
- Assert.fail("Assert.failed to register MXBean : "+e.getMessage());
- }
- }
-
-
- private void safeSleep(long sleep) {
- try {
- Thread.sleep(sleep);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
-
-
-
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
deleted file mode 100644
index 2a67550..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.tasks;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.local.counters.StreamsTaskCounter;
-import org.apache.streams.local.test.processors.PassthroughDatumCounterProcessor;
-import org.apache.streams.local.test.providers.NumericMessageProvider;
-import org.apache.streams.local.test.writer.DatumCounterWriter;
-import org.apache.streams.util.ComponentUtils;
-
-import com.google.common.util.concurrent.Uninterruptibles;
-import org.junit.After;
-import org.junit.Test;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- *
- */
-public class BasicTasksTest {
-
-
- private static final String MBEAN_ID = "test_bean";
- @After
- public void removeLocalMBeans() {
- try {
- ComponentUtils.removeAllMBeansOfDomain("org.apache.streams.local");
- } catch (Exception e) {
- //No op. proceed to next test
- }
- }
-
- @Test
- public void testProviderTask() {
- int numMessages = 100;
- NumericMessageProvider provider = new NumericMessageProvider(numMessages);
- StreamsProviderTask task = new StreamsProviderTask(provider, false, null);
- BlockingQueue<StreamsDatum> outQueue = new LinkedBlockingQueue<>();
- task.addOutputQueue(outQueue);
- //Test that adding input queues to providers is not valid
- BlockingQueue<StreamsDatum> inQueue = createInputQueue(numMessages);
- Exception exp = null;
- try {
- task.addInputQueue(inQueue);
- } catch (UnsupportedOperationException uoe) {
- exp = uoe;
- }
- assertNotNull(exp);
-
- ExecutorService service = Executors.newFixedThreadPool(1);
- service.submit(task);
- int attempts = 0;
- while(outQueue.size() != numMessages) {
- Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
- if(attempts == 10) {
- fail("Provider task failed to output "+numMessages+" in a timely fashion.");
- }
- }
- service.shutdown();
- try {
- if(!service.awaitTermination(10, TimeUnit.SECONDS)){
- service.shutdownNow();
- fail("Service did not terminate.");
- }
- assertTrue("Task should have completed running in allotted time.", service.isTerminated());
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
-
- @Test
- public void testProcessorTask() {
- int numMessages = 100;
- PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor("");
- StreamsProcessorTask task = new StreamsProcessorTask(processor);
- StreamsTaskCounter counter = new StreamsTaskCounter(MBEAN_ID, null, -1);
- task.setStreamsTaskCounter(counter);
- BlockingQueue<StreamsDatum> outQueue = new LinkedBlockingQueue<>();
- BlockingQueue<StreamsDatum> inQueue = createInputQueue(numMessages);
- task.addOutputQueue(outQueue);
- task.addInputQueue(inQueue);
- assertEquals(numMessages, task.getInputQueues().get(0).size());
- ExecutorService service = Executors.newFixedThreadPool(1);
- service.submit(task);
- int attempts = 0;
- while(inQueue.size() != 0 && outQueue.size() != numMessages) {
- Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
- ++attempts;
- if(attempts == 10) {
- fail("Processor task failed to output "+numMessages+" in a timely fashion.");
- }
- }
- task.stopTask();;
- service.shutdown();
- try {
- if(!service.awaitTermination(5, TimeUnit.SECONDS)){
- service.shutdownNow();
- fail("Service did not terminate.");
- }
- assertTrue("Task should have completed running in allotted time.", service.isTerminated());
- } catch (InterruptedException e) {
- fail("Test Interrupted.");
- }
- assertEquals(numMessages, processor.getMessageCount());
- assertEquals(numMessages, counter.getNumReceived());
- assertEquals(numMessages, counter.getNumEmitted());
- assertEquals(0, counter.getNumUnhandledErrors());
- assertEquals(0.0, counter.getErrorRate(), 0.0);
- }
-
- @Test
- public void testWriterTask() {
- int numMessages = 100;
- DatumCounterWriter writer = new DatumCounterWriter("");
- StreamsPersistWriterTask task = new StreamsPersistWriterTask(writer);
- StreamsTaskCounter counter = new StreamsTaskCounter(MBEAN_ID, null, -1);
- task.setStreamsTaskCounter(counter);
- BlockingQueue<StreamsDatum> outQueue = new LinkedBlockingQueue<>();
- BlockingQueue<StreamsDatum> inQueue = createInputQueue(numMessages);
-
- Exception exp = null;
- try {
- task.addOutputQueue(outQueue);
- } catch (UnsupportedOperationException uoe) {
- exp = uoe;
- }
- assertNotNull(exp);
- task.addInputQueue(inQueue);
- assertEquals(numMessages, task.getInputQueues().get(0).size());
- ExecutorService service = Executors.newFixedThreadPool(1);
- service.submit(task);
- int attempts = 0;
- while(inQueue.size() != 0 ) {
- Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
- ++attempts;
- if(attempts == 10) {
- fail("Processor task failed to output "+numMessages+" in a timely fashion.");
- }
- }
- task.stopTask();
- service.shutdown();
- try {
- if(!service.awaitTermination(15, TimeUnit.SECONDS)){
- service.shutdownNow();
- fail("Service did not terminate.");
- }
- assertTrue("Task should have completed running in allotted time.", service.isTerminated());
- } catch (InterruptedException e) {
- fail("Test Interrupted.");
- }
- assertEquals(numMessages, writer.getDatumsCounted());
- assertEquals(numMessages, counter.getNumReceived());
- assertEquals(0, counter.getNumEmitted());
- assertEquals(0, counter.getNumUnhandledErrors());
- assertEquals(0.0, counter.getErrorRate(), 0.0);
- }
-
- @Test
- public void testMergeTask() {
- int numMessages = 100;
- int incoming = 5;
- StreamsMergeTask task = new StreamsMergeTask();
- BlockingQueue<StreamsDatum> outQueue = new LinkedBlockingQueue<>();
- task.addOutputQueue(outQueue);
- for(int i=0; i < incoming; ++i) {
- task.addInputQueue(createInputQueue(numMessages));
- }
- ExecutorService service = Executors.newFixedThreadPool(1);
- service.submit(task);
- int attempts = 0;
- while(outQueue.size() != incoming * numMessages ) {
- Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
- ++attempts;
- if(attempts == 10) {
- assertEquals("Processor task failed to output " + (numMessages * incoming) + " in a timely fashion.", (numMessages * incoming), outQueue.size());
- }
- }
- task.stopTask();
- service.shutdown();
- try {
- if(!service.awaitTermination(5, TimeUnit.SECONDS)){
- service.shutdownNow();
- fail("Service did not terminate.");
- }
- assertTrue("Task should have completed running in allotted time.", service.isTerminated());
- } catch (InterruptedException e) {
- fail("Test Interrupted.");
- }
- }
-
- @Test
- public void testBranching() {
- int numMessages = 100;
- PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor("");
- StreamsProcessorTask task = new StreamsProcessorTask(processor);
- BlockingQueue<StreamsDatum> outQueue1 = new LinkedBlockingQueue<>();
- BlockingQueue<StreamsDatum> outQueue2 = new LinkedBlockingQueue<>();
- BlockingQueue<StreamsDatum> inQueue = createInputQueue(numMessages);
- task.addOutputQueue(outQueue1);
- task.addOutputQueue(outQueue2);
- task.addInputQueue(inQueue);
- assertEquals(numMessages, task.getInputQueues().get(0).size());
- ExecutorService service = Executors.newFixedThreadPool(1);
- service.submit(task);
- int attempts = 0;
- while(inQueue.size() != 0 ) {
- Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
- ++attempts;
- if(attempts == 10) {
- assertEquals("Processor task failed to output "+(numMessages)+" in a timely fashion.", 0, inQueue.size());
- }
- }
- task.stopTask();
-
- service.shutdown();
- try {
- if(!service.awaitTermination(5, TimeUnit.SECONDS)){
- service.shutdownNow();
- fail("Service did not terminate.");
- }
- assertTrue("Task should have completed running in allotted time.", service.isTerminated());
- } catch (InterruptedException e) {
- fail("Test Interrupted.");
- }
- assertEquals(numMessages, processor.getMessageCount());
- assertEquals(numMessages, outQueue1.size());
- assertEquals(numMessages, outQueue2.size());
- }
-
- @Test
- public void testBranchingSerialization() {
- int numMessages = 1;
- PassthroughDatumCounterProcessor processor = new PassthroughDatumCounterProcessor("");
- StreamsProcessorTask task = new StreamsProcessorTask(processor);
- BlockingQueue<StreamsDatum> outQueue1 = new LinkedBlockingQueue<>();
- BlockingQueue<StreamsDatum> outQueue2 = new LinkedBlockingQueue<>();
- BlockingQueue<StreamsDatum> inQueue = createInputQueue(numMessages);
- task.addOutputQueue(outQueue1);
- task.addOutputQueue(outQueue2);
- task.addInputQueue(inQueue);
- ExecutorService service = Executors.newFixedThreadPool(1);
- service.submit(task);
- int attempts = 0;
- while(inQueue.size() != 0 ) {
- Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
- ++attempts;
- if(attempts == 10) {
- assertEquals("Processor task failed to output "+(numMessages)+" in a timely fashion.", 0, inQueue.size());
- }
- }
- task.stopTask();
-
- service.shutdown();
- try {
- if(!service.awaitTermination(5, TimeUnit.SECONDS)){
- service.shutdownNow();
- fail("Service did not terminate.");
- }
- assertTrue("Task should have completed running in allotted time.", service.isTerminated());
- } catch (InterruptedException e) {
- fail("Test Interrupted.");
- }
- assertEquals(numMessages, processor.getMessageCount());
- assertEquals(numMessages, outQueue1.size());
- assertEquals(numMessages, outQueue2.size());
- StreamsDatum datum1 = outQueue1.poll();
- StreamsDatum datum2 = outQueue2.poll();
- assertNotNull(datum1);
- assertEquals(datum1, datum2);
- datum1.setDocument("a");
- assertNotEquals(datum1, datum2);
- }
-
- private BlockingQueue<StreamsDatum> createInputQueue(int numDatums) {
- BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>();
- for(int i=0; i < numDatums; ++i) {
- queue.add(new StreamsDatum(i));
- }
- return queue;
- }
-
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/StreamsProviderTaskTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/StreamsProviderTaskTest.java
deleted file mode 100644
index 782e232..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/StreamsProviderTaskTest.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.tasks;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.util.ComponentUtils;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.Queue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.atLeast;
-import static org.mockito.Mockito.atMost;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-/**
- * Tests the StreamsProviderTask.
- */
-public class StreamsProviderTaskTest {
-
- protected StreamsProvider mockProvider;
- protected ExecutorService pool;
-
- @Before
- public void setup() {
- mockProvider = mock(StreamsProvider.class);
- pool = Executors.newFixedThreadPool(1);
- }
-
- @After
- public void removeLocalMBeans() {
- try {
- ComponentUtils.removeAllMBeansOfDomain("org.apache.streams.local");
- } catch (Exception e) {
- //No op. proceed to next test
- }
- }
-
- @Test
- public void runPerpetual() {
- StreamsProviderTask task = new StreamsProviderTask(mockProvider, true, null);
- when(mockProvider.isRunning()).thenReturn(true);
- when(mockProvider.readCurrent()).thenReturn(new StreamsResultSet(new LinkedBlockingQueue<StreamsDatum>()));
- task.setTimeout(500);
- task.setSleepTime(10);
- task.run();
- //Setting this to at least 2 means that it was correctly set to perpetual mode
- verify(mockProvider, atLeast(2)).readCurrent();
- verify(mockProvider, atMost(1)).prepare(null);
- }
-
- @Test
- public void flushes() {
- BlockingQueue<StreamsDatum> out = new LinkedBlockingQueue<>();
- StreamsProviderTask task = new StreamsProviderTask(mockProvider, true, null);
- when(mockProvider.isRunning()).thenReturn(true);
- when(mockProvider.readCurrent()).thenReturn(new StreamsResultSet(getQueue(3)));
- task.setTimeout(100);
- task.setSleepTime(10);
- task.getOutputQueues().add(out);
- task.run();
- assertThat(out.size(), is(equalTo(3)));
- }
-
- protected Queue<StreamsDatum> getQueue(int numElems) {
- Queue<StreamsDatum> results = new LinkedBlockingQueue<>();
- for(int i=0; i<numElems; i++) {
- results.add(new StreamsDatum(Math.random()));
- }
- return results;
- }
-
- @Test
- public void runNonPerpetual() {
- StreamsProviderTask task = new StreamsProviderTask(mockProvider, false, null);
- when(mockProvider.isRunning()).thenReturn(true);
- when(mockProvider.readCurrent()).thenReturn(new StreamsResultSet(new LinkedBlockingQueue<StreamsDatum>()));
- task.setTimeout(500);
- task.setSleepTime(10);
- task.run();
- //In read current mode, this should only be called 1 time
- verify(mockProvider, atLeast(1)).readCurrent();
- verify(mockProvider, atMost(1)).prepare(null);
- }
-
- @Test
- public void stoppable() throws InterruptedException {
- StreamsProviderTask task = new StreamsProviderTask(mockProvider, true, null);
- when(mockProvider.isRunning()).thenReturn(true);
- when(mockProvider.readCurrent()).thenReturn(new StreamsResultSet(new LinkedBlockingQueue<StreamsDatum>()));
- task.setTimeout(-1);
- task.setSleepTime(10);
- Future<?> taskResult = pool.submit(task);
-
- //After a few milliseconds, tell the task that it is to stop and wait until it says it isn't or a timeout happens
- int count = 0;
- do {
- Thread.sleep(100);
- if(count == 0) {
- task.stopTask();
- }
- } while(++count < 10 && !taskResult.isDone());
- verifyNotRunning(task, taskResult);
-
- }
-
- @Test
- public void earlyException() throws InterruptedException {
- StreamsProviderTask task = new StreamsProviderTask(mockProvider, true, null);
- when(mockProvider.isRunning()).thenReturn(true);
- doThrow(new RuntimeException()).when(mockProvider).prepare(null);
- task.setTimeout(-1);
- task.setSleepTime(10);
- Future<?> taskResult = pool.submit(task);
- int count = 0;
- while(++count < 10 && !taskResult.isDone()) {
- Thread.sleep(100);
- }
- verifyNotRunning(task, taskResult);
- }
-
- protected void verifyNotRunning(StreamsProviderTask task, Future<?> taskResult) {
- //Make sure the task is reporting that it is complete and that the run method returned
- if(taskResult.isDone()) {
- assertThat(task.isRunning(), is(false));
- } else {
- ComponentUtils.shutdownExecutor(pool, 0, 10);
- fail();
- }
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/DoNothingProcessor.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/DoNothingProcessor.java
deleted file mode 100644
index 31a83ec..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/DoNothingProcessor.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.test.processors;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * Created by rebanks on 2/20/14.
- */
-public class DoNothingProcessor implements StreamsProcessor {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DoNothingProcessor.class);
-
- public final static String STREAMS_ID = "DoNothingProcessor";
-
- List<StreamsDatum> result;
-
- public DoNothingProcessor() {
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- this.result = new LinkedList<StreamsDatum>();
- result.add(entry);
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
- LOGGER.debug("Processor clean up!");
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/PassthroughDatumCounterProcessor.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/PassthroughDatumCounterProcessor.java
deleted file mode 100644
index 43343e5..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/PassthroughDatumCounterProcessor.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.test.processors;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- *
- */
-public class PassthroughDatumCounterProcessor implements StreamsProcessor {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(PassthroughDatumCounterProcessor.class);
-
- public final static String STREAMS_ID = "PassthroughDatumCounterProcessor";
-
- /**
- * Set of all ids that have been claimed. Ensures all instances are assigned unique ids
- */
- public static Set<Integer> CLAIMED_ID = new HashSet<Integer>();
- /**
- * Random instance to generate ids
- */
- public static final Random RAND = new Random();
- /**
- * Set of instance ids that received data. Usefully for testing parrallelization is actually working.
- */
- public final static Set<Integer> SEEN_DATA = new HashSet<Integer>();
- /**
- * The total count of data seen by a all instances of a processor.
- */
- public static final ConcurrentHashMap<String, AtomicLong> COUNTS = new ConcurrentHashMap<>();
-
- private int count = 0;
- private int id;
- private String procId;
-
- public PassthroughDatumCounterProcessor(String procId) {
- this.procId = procId;
- }
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- ++this.count;
- List<StreamsDatum> result = new LinkedList<StreamsDatum>();
- result.add(entry);
- synchronized (SEEN_DATA) {
- SEEN_DATA.add(this.id);
- }
- return result;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- synchronized (CLAIMED_ID) {
- this.id = RAND.nextInt();
- while(!CLAIMED_ID.add(this.id)) {
- this.id = RAND.nextInt();
- }
- }
- }
-
- @Override
- public void cleanUp() {
- LOGGER.debug("Clean up {}", this.procId);
- synchronized (COUNTS) {
- AtomicLong count = COUNTS.get(this.procId);
- if(count == null) {
- COUNTS.put(this.procId, new AtomicLong(this.count));
- } else {
- count.addAndGet(this.count);
- }
- }
- LOGGER.debug("{}\t{}", this.procId, this.count);
- }
-
- public int getMessageCount() {
- return this.count;
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/SlowProcessor.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/SlowProcessor.java
deleted file mode 100644
index 9ad61ab..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/processors/SlowProcessor.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.test.processors;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- */
-public class SlowProcessor implements StreamsProcessor {
-
- public final static String STREAMS_ID = "DoNothingProcessor";
-
- @Override
- public String getId() {
- return STREAMS_ID;
- }
-
- @Override
- public List<StreamsDatum> process(StreamsDatum entry) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- return Stream.of(entry).collect(Collectors.toList());
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
-
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/providers/EmptyResultSetProvider.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/providers/EmptyResultSetProvider.java
deleted file mode 100644
index b85ad4a..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/providers/EmptyResultSetProvider.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.test.providers;
-
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-
-import org.joda.time.DateTime;
-
-import java.math.BigInteger;
-import java.util.concurrent.LinkedBlockingQueue;
-
-/**
- * Provides new, empty instances of result set.
- */
-public class EmptyResultSetProvider implements StreamsProvider {
-
- @Override
- public String getId() {
- return "EmptyResultSetProvider";
- }
-
- @Override
- public void startStream() {
- //NOP
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- return new StreamsResultSet(new LinkedBlockingQueue<>());
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return new StreamsResultSet(new LinkedBlockingQueue<>());
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return new StreamsResultSet(new LinkedBlockingQueue<>());
- }
-
- @Override
- public boolean isRunning() {
- return true;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- //NOP
- }
-
- @Override
- public void cleanUp() {
- //NOP
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/providers/NumericMessageProvider.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/providers/NumericMessageProvider.java
deleted file mode 100644
index 21f37ba..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/providers/NumericMessageProvider.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.test.providers;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-
-import org.joda.time.DateTime;
-
-import java.math.BigInteger;
-import java.util.Queue;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-/**
- * Test StreamsProvider that sends out StreamsDatums numbered from 0 to numMessages.
- */
-public class NumericMessageProvider implements StreamsProvider {
-
- @Override
- public String getId() {
- return "NumericMessageProvider";
- }
-
- private static final int DEFAULT_BATCH_SIZE = 100;
-
- private int numMessages;
- private BlockingQueue<StreamsDatum> data;
- private volatile boolean complete = false;
-
- public NumericMessageProvider(int numMessages) {
- this.numMessages = numMessages;
- }
-
- @Override
- public void startStream() {
- this.data = constructQueue();
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- int batchSize = 0;
- Queue<StreamsDatum> batch = new LinkedBlockingQueue<>();
- try {
- while (!this.data.isEmpty() && batchSize < DEFAULT_BATCH_SIZE) {
- batch.add(this.data.take());
- ++batchSize;
- }
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
-// System.out.println("******************\n**\tBatchSize="+batch.size()+"\n******************");
- this.complete = batch.isEmpty() && this.data.isEmpty();
- return new StreamsResultSet(batch);
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return new StreamsResultSet(constructQueue());
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return new StreamsResultSet(constructQueue());
- }
-
- @Override
- public boolean isRunning() {
- return !this.complete;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.data = constructQueue();
- }
-
- @Override
- public void cleanUp() {
-
- }
-
- private BlockingQueue<StreamsDatum> constructQueue() {
- BlockingQueue<StreamsDatum> datums = new ArrayBlockingQueue<>(numMessages);
- for(int i=0;i<numMessages;i++) {
- datums.add(new StreamsDatum(i));
- }
- return datums;
- }
-}
-
-
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DatumCounterWriter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DatumCounterWriter.java
deleted file mode 100644
index 19a2ad8..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DatumCounterWriter.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.test.writer;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- *
- */
-public class DatumCounterWriter implements StreamsPersistWriter{
-
- @Override
- public String getId() {
- return "DatumCounterWriter";
- }
-
- /**
- * Set of all ids that have been claimed. Ensures all instances are assigned unique ids
- */
- public static Set<Integer> CLAIMED_ID = new HashSet<>();
- /**
- * Random instance to generate ids
- */
- public static final Random RAND = new Random();
- /**
- * Set of instance ids that received data. Usefully for testing parrallelization is actually working.
- */
- public final static Set<Integer> SEEN_DATA = Collections.newSetFromMap(new ConcurrentHashMap<Integer, Boolean>());
- /**
- * The total count of data seen by a all instances of a processor.
- */
- public static final ConcurrentHashMap<String, AtomicLong> COUNTS = new ConcurrentHashMap<>();
- /**
- * The documents received
- */
- public static final ConcurrentHashMap<String, List<Object>> RECEIVED = new ConcurrentHashMap<>();
-
- private int counter = 0;
- private String writerId;
- private Integer id;
-
- public DatumCounterWriter(String writerId) {
- this.writerId = writerId;
- }
-
- @Override
- public void write(StreamsDatum entry) {
- ++this.counter;
- SEEN_DATA.add(this.id);
- synchronized (RECEIVED) {
- List<Object> documents = RECEIVED.get(this.writerId);
- if(documents == null) {
- List<Object> docs = new LinkedList<>();
- docs.add(entry.getDocument());
- RECEIVED.put(this.writerId, docs);
- } else {
- documents.add(entry.getDocument());
- }
- }
- }
-
- @Override
- public void prepare(Object configurationObject) {
- synchronized (CLAIMED_ID) {
- this.id = RAND.nextInt();
- while(!CLAIMED_ID.add(this.id)) {
- this.id = RAND.nextInt();
- }
- }
- }
-
- @Override
- public void cleanUp() {
- synchronized (COUNTS) {
- AtomicLong count = COUNTS.get(this.writerId);
- if(count == null) {
- COUNTS.put(this.writerId, new AtomicLong(this.counter));
- } else {
- count.addAndGet(this.counter);
- }
- }
- }
-
- public int getDatumsCounted() {
- return this.counter;
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DoNothingWriter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DoNothingWriter.java
deleted file mode 100644
index 48f4b68..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/DoNothingWriter.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.test.writer;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Created by rebanks on 2/20/14.
- */
-public class DoNothingWriter implements StreamsPersistWriter {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(DoNothingWriter.class);
-
- @Override
- public String getId() {
- return "DoNothingWriter";
- }
-
- @Override
- public void write(StreamsDatum entry) {
-
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
- LOGGER.debug("Writer Clean Up!");
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/SystemOutWriter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/SystemOutWriter.java
deleted file mode 100644
index 2711ae1..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/test/writer/SystemOutWriter.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.local.test.writer;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Created by rebanks on 2/20/14.
- */
-public class SystemOutWriter implements StreamsPersistWriter {
-
- private final static Logger LOGGER = LoggerFactory.getLogger(SystemOutWriter.class);
-
- @Override
- public String getId() {
- return "SystemOutWriter";
- }
-
- @Override
- public void write(StreamsDatum entry) {
- System.out.println(entry.document);
- }
-
- @Override
- public void prepare(Object configurationObject) {
-
- }
-
- @Override
- public void cleanUp() {
- LOGGER.debug("Clean up called writer!");
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/ExpectedDatumsPersistWriter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/ExpectedDatumsPersistWriter.java
deleted file mode 100644
index 16b98c4..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/ExpectedDatumsPersistWriter.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.test.component;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsPersistWriter;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Scanner;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
-/**
- * Created by rebanks on 2/27/14.
- */
-public class ExpectedDatumsPersistWriter implements StreamsPersistWriter{
-
- @Override
- public String getId() {
- return "ExpectedDatumsPersistWriter";
- }
-
- private StreamsDatumConverter converter;
- private String fileName;
- private List<StreamsDatum> expectedDatums;
- private int counted = 0;
- private int expectedSize = 0;
-
- public ExpectedDatumsPersistWriter(StreamsDatumConverter converter, String filePathInResources) {
- this.converter = converter;
- this.fileName = filePathInResources;
- }
-
-
-
- @Override
- public void write(StreamsDatum entry) {
- int index = this.expectedDatums.indexOf(entry);
- assertNotEquals("Datum not expected. "+entry.toString(), -1, index);
- this.expectedDatums.remove(index);
- ++this.counted;
- }
-
- @Override
- public void prepare(Object configurationObject) {
- Scanner scanner = new Scanner(ExpectedDatumsPersistWriter.class.getResourceAsStream(this.fileName));
- this.expectedDatums = new LinkedList<StreamsDatum>();
- while(scanner.hasNextLine()) {
- this.expectedDatums.add(this.converter.convert(scanner.nextLine()));
- }
- this.expectedSize = this.expectedDatums.size();
- }
-
- @Override
- public void cleanUp() {
- assertEquals("Did not received the expected number of StreamsDatums", this.expectedSize, this.counted);
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/FileReaderProvider.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/FileReaderProvider.java
deleted file mode 100644
index 632d079..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/FileReaderProvider.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.test.component;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
-
-import org.joda.time.DateTime;
-
-import java.math.BigInteger;
-import java.util.Queue;
-import java.util.Scanner;
-import java.util.concurrent.LinkedBlockingQueue;
-
-/**
- * FOR TESTING PURPOSES ONLY.
- *
- * The StreamProvider reads from a File or InputStream. Each line of the file will be emitted as the document of a
- * streams datum.
- *
- */
-public class FileReaderProvider implements StreamsProvider {
-
- private String fileName;
- private Scanner scanner;
- private StreamsDatumConverter converter;
-
- public FileReaderProvider(String filePathInResources, StreamsDatumConverter converter) {
- this.fileName = filePathInResources;
- this.converter = converter;
- }
-
- @Override
- public String getId() {
- return "FileReaderProvider";
- }
-
- @Override
- public void startStream() {
-
- }
-
- @Override
- public StreamsResultSet readCurrent() {
- return new StreamsResultSet(constructQueue(this.scanner));
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean isRunning() {
- return this.scanner != null && this.scanner.hasNextLine();
- }
-
- @Override
- public void prepare(Object configurationObject) {
- this.scanner = new Scanner(FileReaderProvider.class.getResourceAsStream(this.fileName));
- }
-
- @Override
- public void cleanUp() {
- if(this.scanner!= null) {
- this.scanner.close();
- this.scanner = null;
- }
- }
-
- private Queue<StreamsDatum> constructQueue(Scanner scanner) {
- Queue<StreamsDatum> data = new LinkedBlockingQueue<>();
- while(scanner.hasNextLine()) {
- data.add(converter.convert(scanner.nextLine()));
- }
- cleanUp();
- return data;
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StreamsDatumConverter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StreamsDatumConverter.java
deleted file mode 100644
index 9172167..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StreamsDatumConverter.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.test.component;
-
-import org.apache.streams.core.StreamsDatum;
-
-import java.io.Serializable;
-
-/**
- * Created by rebanks on 2/27/14.
- */
-public interface StreamsDatumConverter extends Serializable {
-
- public StreamsDatum convert(String s);
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StringToDocumentConverter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StringToDocumentConverter.java
deleted file mode 100644
index 3727aa1..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/StringToDocumentConverter.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.test.component;
-
-import org.apache.streams.core.StreamsDatum;
-
-/**
- * Created by rebanks on 2/28/14.
- */
-public class StringToDocumentConverter implements StreamsDatumConverter {
-
- @Override
- public StreamsDatum convert(String s) {
- return new StreamsDatum(s);
- }
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestComponentsLocalStream.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestComponentsLocalStream.java
deleted file mode 100644
index 5154ea3..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestComponentsLocalStream.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.test.component.tests;
-
-import org.apache.streams.local.builders.LocalStreamBuilder;
-import org.apache.streams.test.component.ExpectedDatumsPersistWriter;
-import org.apache.streams.test.component.FileReaderProvider;
-import org.apache.streams.test.component.StringToDocumentConverter;
-import org.apache.streams.util.ComponentUtils;
-
-import org.junit.After;
-import org.junit.Test;
-
-/**
- *
- */
-public class TestComponentsLocalStream {
-
- @After
- public void removeLocalMBeans() {
- try {
- ComponentUtils.removeAllMBeansOfDomain("org.apache.streams.local");
- } catch (Exception e) {
- //No op. proceed to next test
- }
- }
-
- @Test
- public void testLocalStreamWithComponent() {
- LocalStreamBuilder builder = new LocalStreamBuilder();
- builder.newReadCurrentStream("provider", new FileReaderProvider("/TestFile.txt",
- new StringToDocumentConverter()));
- builder.addStreamsPersistWriter("writer", new ExpectedDatumsPersistWriter(new StringToDocumentConverter(),
- "/TestFile.txt"), 1, "provider")
- .start();
- }
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestExpectedDatumsPersitWriter.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestExpectedDatumsPersitWriter.java
deleted file mode 100644
index 0535295..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestExpectedDatumsPersitWriter.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.test.component.tests;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.test.component.ExpectedDatumsPersistWriter;
-import org.apache.streams.test.component.StringToDocumentConverter;
-import org.apache.streams.util.ComponentUtils;
-
-import org.junit.After;
-import org.junit.Test;
-
-/**
- *
- */
-public class TestExpectedDatumsPersitWriter {
-
- private static final StreamsDatum[] INPUT_DATUMS = new StreamsDatum[] {
- new StreamsDatum("Document1"),
- new StreamsDatum("Document2"),
- new StreamsDatum("Document3"),
- new StreamsDatum("Document4")
-// Uncomment to prove failures occur, or comment out a datum above
-// ,new StreamsDatum("Document5")
- };
-
- @After
- public void removeLocalMBeans() {
- try {
- ComponentUtils.removeAllMBeansOfDomain("org.apache.streams.local");
- } catch (Exception e) {
- //No op. proceed to next test
- }
- }
-
- @Test
- public void testExpectedDatumsPersistWriterFileName() {
- testDatums(new ExpectedDatumsPersistWriter(new StringToDocumentConverter(), "/TestFile.txt"));
- }
-
-
-
- private void testDatums(ExpectedDatumsPersistWriter writer) {
- writer.prepare(null);
- for(StreamsDatum datum : INPUT_DATUMS) {
- writer.write(datum);
- }
- writer.cleanUp();
- }
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestFileReaderProvider.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestFileReaderProvider.java
deleted file mode 100644
index a2b7bba..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/test/component/tests/TestFileReaderProvider.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.test.component.tests;
-
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsResultSet;
-import org.apache.streams.test.component.FileReaderProvider;
-import org.apache.streams.test.component.StringToDocumentConverter;
-import org.apache.streams.util.ComponentUtils;
-
-import org.junit.After;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- *
- */
-public class TestFileReaderProvider {
-
- @After
- public void removeLocalMBeans() {
- try {
- ComponentUtils.removeAllMBeansOfDomain("org.apache.streams.local");
- } catch (Exception e) {
- //No op. proceed to next test
- }
- }
-
- @Test
- public void testFileReaderProviderFileName() {
- String fileName = "/TestFile.txt";
- FileReaderProvider provider = new FileReaderProvider(fileName, new StringToDocumentConverter());
- provider.prepare(null);
- StreamsResultSet resultSet = provider.readCurrent();
- int count = 0;
- for(StreamsDatum datum : resultSet) {
- ++count;
- }
- assertEquals(4, count);
- provider.cleanUp();
- }
-
-
-
-
-
-
-}
diff --git a/streams-runtimes/streams-runtime-local/src/test/resources/TestFile.txt b/streams-runtimes/streams-runtime-local/src/test/resources/TestFile.txt
deleted file mode 100644
index 9384ee5..0000000
--- a/streams-runtimes/streams-runtime-local/src/test/resources/TestFile.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-Document1
-Document2
-Document3
-Document4
\ No newline at end of file
diff --git a/streams-schemas/pom.xml b/streams-schemas/pom.xml
deleted file mode 100644
index df745bf..0000000
--- a/streams-schemas/pom.xml
+++ /dev/null
@@ -1,40 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>apache-streams</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-schemas</artifactId>
- <name>${project.artifactId}</name>
- <packaging>pom</packaging>
-
- <description>Aggregator for schema modules</description>
-
- <modules>
- <module>streams-schema-activitystreams</module>
- <module>streams-schema-activitystreams2</module>
- </modules>
-
-</project>
diff --git a/streams-schemas/streams-schema-activitystreams/pom.xml b/streams-schemas/streams-schema-activitystreams/pom.xml
deleted file mode 100644
index a376206..0000000
--- a/streams-schemas/streams-schema-activitystreams/pom.xml
+++ /dev/null
@@ -1,114 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schemas</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-schema-activitystreams</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Activity Streams schemas</description>
-
- <dependencies>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- <version>${slf4j.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>com.networknt</groupId>
- <artifactId>json-schema-validator</artifactId>
- <type>jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-resource</goal>
- </goals>
- <configuration>
- <resources>
- <resource>
- <directory>${project.basedir}/src/main/jsonschema</directory>
- </resource>
- <resource>
- <directory>src/main/xmlschema</directory>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <includes>
- <include>**/*.json</include>
- </includes>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/activity.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/activity.json
deleted file mode 100644
index 89ddeec..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/activity.json
+++ /dev/null
@@ -1,108 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "activity",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "An activity construct recounts what an actor did to an object in the past. If there is no actor it simply describes the change.",
- "additionalProperties": true,
- "properties": {
- "id": {
- "type": "string",
- "description": "Uniquely identifies each activity within the service",
- "required": true
- },
- "actor": {
- "type": "object",
- "required": true,
- "description": "Describes the entity that performed the activity. An activity MUST contain one actor property whose value is a single Object.",
- "javaInterfaces": ["java.io.Serializable"],
- "$ref": "./object.json"
- },
- "verb": {
- "title": "verb",
- "type": "string",
- "default": "post",
- "required": true,
- "description": "Identifies the action that the activity describes. An activity MUST contain a verb property whose value is a JSON String that is non-empty and matches either the \"isegment-nz-nc\" or the \"IRI\" production in [RFC3987]. Note that the use of a relative reference other than a simple name is not allowed."
- },
- "object": {
- "type": "object",
- "required": true,
- "description": "Describes the primary object of the activity. For instance, in the activity, \"John saved a movie to his wishlist\", the object of the activity is \"movie\". An activity SHOULD contain an object property whose value is a single Object. If the object property is not contained, the primary object of the activity MAY be implied by context.",
- "$ref": "./object.json"
- },
- "target": {
- "type": "object",
- "description": "Describes the target of the activity. The precise meaning of the activity's target is dependent on the activities verb, but will often be the object the English preposition \"to\". For instance, in the activity, \"John saved a movie to his wishlist\", the target of the activity is \"wishlist\". The activity target MUST NOT be used to identity an indirect object that is not a target of the activity. An activity MAY contain a target property whose value is a sin [...]
- "$ref": "./object.json"
- },
- "published": {
- "type": "string",
- "description": "The date and time at which the activity occurred. It is important to note that this is not necessarily the same as the time at which the activity was published. An activity MUST contain a postedTime property.",
- "format": "date-time"
- },
- "updated": {
- "type": "string",
- "description": "The date and time at which a previously published activity has been modified. An Activity MAY contain an updatedTime property",
- "format": "date-time"
- },
- "generator": {
- "type": "object",
- "description": "Describes the application that generated the activity. An activity MAY contain a generator property whose value is a single Object.",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {
- "$ref": "./object.json"
- }
- },
- "icon": {
- "type": "object",
- "description": "An IRI[RFC3987] identifying an image resource provides a visual representation of the activity, intended for human consumption. The image SHOULD have an aspect ratio of one (horizontal) to one (vertical) and SHOULD be suitable for presentation at a small size. An activity MAY have an icon property",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {
- "$ref": "./media_link.json"
- }
- },
- "provider": {
- "type": "object",
- "description": "Describes the application that published the activity. Note that this is not necessarily the same entity that generated the activity. An activity MAY contain a provider property whose value is a single Object",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {
- "$ref": "./object.json"
- }
- },
- "title": {
- "type": "string",
- "description": "Natural-language title or headline for the activity encoded as a single JSON String containing HTML markup. An activity MAY contain a title property",
- "format": "html"
- },
- "content": {
- "type": "string",
- "description": "Natural-language description of the activity encoded as a single JSON String containing HTML markup. Visual elements such as thumbnail images MAY be included. An activity MAY contain a content property",
- "format": "html"
- },
- "url": {
- "type": "string",
- "description": "An IRI [RFC3987] identifying a resource providing an HTML representation of the activity. An activity MAY contain a url property",
- "format": "url"
- },
- "links": {
- "type": "array",
- "description": "Links between this object and other resources as defined in Web Linking",
- "note": "Tell JSON schema team to not put links inside http://json-schema.org/hyper-schema#properties",
- "items": {
- "type": "string"
- }
- }
- },
- "links": [
- {
- "rel": "describedby",
- "href": "./verbs/{verb}.json"
- }
- ]
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/collection.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/collection.json
deleted file mode 100644
index 5606c0d..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/collection.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "collection",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "A collection is a generic list of Objects of any object type. The objectType of each item in the collection MAY be omitted if the type of object can be established through context. The collection is used primarily as the root of an Activity Streams document, but can be used as the value of extension properties in a variety of situations. ",
- "properties": {
- "url": {
- "type": "string",
- "description": "An IRI [RFC3987] referencing a JSON document containing the full listing of objects in the collection."
- },
- "totalItems": {
- "type": "integer",
- "description": "Non-negative integer specifying the total number of activities within the stream. The Stream serialization MAY contain a count property."
- },
- "items": {
- "type": "array",
- "required": true,
- "description": "An array containing a listing of Objects of any object type. If used in combination with the url property, the items array can be used to provide a subset of the objects that may be found in the resource identified by the url.",
- "items": {
- "type": "object",
- "extends": {
- "$ref": "./object.json"
- }
- }
- },
- "links": {
- "type": "array",
- "optional": true,
- "description": "Links between an this object and other resources as defined in Web Linking",
- "extends": {
- "$ref": "http://json-schema.org/links#properties"
- }
- }
- },
- "links": [
- {
- "rel": "describedby",
- "href": "http://json-schema.org/schema"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/media_link.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/media_link.json
deleted file mode 100644
index c5d6ce0..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/media_link.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "media_link",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "Visual representation of an object in the form of an image, video or embedded HTML fragments",
- "properties": {
- "duration": {
- "title": "duration",
- "type": "number",
- "description": "A hint to the consumer about the length, in seconds, of the media resource identified by the url property. A media link MAY contain a \"duration\" property when the target resource is a time-based media item such as an audio or video."
- },
- "height": {
- "title": "height",
- "type": "integer",
- "description": "A hint to the consumer about the height, in pixels, of the media resource identified by the url property. A media link MAY contain a height property when the target resource is a visual media item such as an image, video or embeddable HTML page."
- },
- "width": {
- "title": "width",
- "type": "integer",
- "description": "A hint to the consumer about the width, in pixels, of the media resource identified by the url property. A media link MAY contain a width property when the target resource is a visual media item such as an image, video or embeddable HTML page."
- },
- "url": {
- "type": "string",
- "required": true,
- "description": "The IRI of the media resource being linked. A media link MUST have a url property."
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/object.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/object.json
deleted file mode 100644
index c00362f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/object.json
+++ /dev/null
@@ -1,99 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "object",
- "javaType": "org.apache.streams.pojo.json.ActivityObject",
- "javaInterfaces": ["java.io.Serializable"],
- "description": "Basic object on the web. The only required property is the id",
- "properties": {
- "id": {
- "type": "string",
- "description": "Provides a permanent, universally unique identifier for the object in the form of an absolute IRI [RFC3987]. An object SHOULD contain a single id property. If an object does not contain an id property, consumers MAY use the value of the url property as a less-reliable, non-unique identifier.",
- "required": true
- },
- "image": {
- "format": "image",
- "type": "object",
- "description": "Description of a resource providing a visual representation of the object, intended for human consumption. An object MAY contain an image property whose value is a Media Link.",
- "extends": {
- "$ref": "./media_link.json"
- }
- },
- "displayName": {
- "type": "string",
- "description": "A natural-language, human-readable and plain-text name for the object. HTML markup MUST NOT be included. An object MAY contain a displayName property. If the object does not specify an objectType property, the object SHOULD specify a displayName"
- },
- "summary": {
- "type": "string",
- "description": "Natural-language summary of the object encoded as a single JSON String containing HTML markup. Visual elements such as thumbnail images MAY be included. An activity MAY contain a summary property"
- },
- "content": {
- "type": "string",
- "description": "Natural-language description of the object encoded as a single JSON String containing HTML markup. Visual elements such as thumbnail images MAY be included. An object MAY contain a content property"
- },
- "url": {
- "type": "string",
- "format": "url",
- "description": "An IRI [RFC3987] identifying a resource providing an HTML representation of the object. An object MAY contain a url property"
- },
- "objectType": {
- "type": "string",
- "description": "Identifies the type of object. An object MAY contain an objectType property whose value is a JSON String that is non-empty and matches either the \"isegment-nz-nc\" or the \"IRI\" production in [RFC3987]. Note that the use of a relative reference other than a simple name is not allowed. If no objectType property is contained, the object has no specific type."
- },
- "author": {
- "type": "object",
- "description": "Describes the entity that created or authored the object. An object MAY contain a single author property whose value is an Object of any type. Note that the author field identifies the entity that created the object and does not necessarily identify the entity that published the object. For instance, it may be the case that an object created by one person is posted and published to a system by an entirely different entity",
- "javaInterfaces": ["java.io.Serializable"],
- "extends": {
- "$ref": "./object.json"
- }
- },
- "published": {
- "type": "string",
- "description": "[RFC3339] date-time. The date and time at which the object was published. An object MAY contain a published property",
- "format": "date-time"
- },
- "updated": {
- "type": "string",
- "description": "[RFC3339] date-time. The date and time at which a previously published object has been modified. An Object MAY contain an updated property.",
- "format": "date-time"
- },
- "attachments": {
- "title": "Related objects",
- "description": "A collection of one or more additional, associated objects, similar to the concept of attached files in an email message. An object MAY have an attachedObjects property whose value is a JSON Array of Objects.",
- "type": "array",
- "items": {
- "type": "object",
- "javaType": "org.apache.streams.pojo.json.Attachment",
- "javaInterfaces": ["java.io.Serializable"],
- "$ref": "./object.json"
- }
- },
- "upstreamDuplicates": {
- "type": "array",
- "description": "A JSON Array of one or more absolute IRI's [RFC3987] identifying objects that duplicate this object's content. An object SHOULD contain an upstreamDuplicates property when a publisher is knowingly duplicating with a new ID the content from another object. This MAY be used as a hint for consumers to use when resolving duplicates between objects received from different sources",
- "items": {
- "type": "string"
- }
- },
- "downstreamDuplicates": {
- "type": "array",
- "description": "A JSON Array of one or more absolute IRI's [RFC3987] identifying objects that duplicate this object's content. An object SHOULD contain a downstreamDuplicates property when there are known objects, possibly in a different system, that duplicate the content in this object. This MAY be used as a hint for consumers to use when resolving duplicates between objects received from different sources.",
- "items": {
- "type": "string"
- }
- }
- },
- "additionalProperties": true,
- "links": [
- {
- "rel": "describedby",
- "href": "./objectType/{objectType}.json"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/alert.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/alert.json
deleted file mode 100644
index d089699..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/alert.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "alert",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "alert"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/application.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/application.json
deleted file mode 100644
index a5c2bdc..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/application.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "application",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "application"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/article.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/article.json
deleted file mode 100644
index 5bbfd9f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/article.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "article",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "article"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/audio.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/audio.json
deleted file mode 100644
index 9c0d33a..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/audio.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "audio",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "audio"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/badge.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/badge.json
deleted file mode 100644
index bb66203..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/badge.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "badge",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "badge"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/binary.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/binary.json
deleted file mode 100644
index 0bbfb87..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/binary.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "binary",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "binary"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/bookmark.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/bookmark.json
deleted file mode 100644
index d052898..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/bookmark.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "bookmark",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "bookmark"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/comment.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/comment.json
deleted file mode 100644
index 4c658ae..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/comment.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "comment",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "comment"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/device.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/device.json
deleted file mode 100644
index e056c69..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/device.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "device",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "device"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/event.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/event.json
deleted file mode 100644
index 82993ee..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/event.json
+++ /dev/null
@@ -1,51 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "event",
- "description": "xCal fromat for vevent",
- "extends": [
- {
- "$ref": "../object.json"
- },
- {
- "$ref": "http://www.json-schema.org/calendar"
- }
- ],
- "properties": {
- "objectType": {
- "type": "string",
- "default": "event"
- },
- "attendedBy": {
- "type": "../collection.json"
- },
- "attending": {
- "type": "../collection.json"
- },
- "endTime": {
- "type": "string",
- "format": "date-time"
- },
- "invited": {
- "type": "../collection.json"
- },
- "maybeAttending": {
- "type": "../collection.json"
- },
- "notAttendedBy": {
- "type": "../collection.json"
- },
- "notAttending": {
- "type": "../collection.json"
- },
- "startTime": {
- "type": "string",
- "format": "date-time"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/file.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/file.json
deleted file mode 100644
index d39fe63..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/file.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "file",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "file"
- },
- "fileUrl": {
- "type": "string"
- },
- "mimeType": {
- "type": "string"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/folder.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/folder.json
deleted file mode 100644
index 4be89f3..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/folder.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "folder",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "folder"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/game.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/game.json
deleted file mode 100644
index 434f152..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/game.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "game",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "game"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/group.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/group.json
deleted file mode 100644
index cc30c29..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/group.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "group",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "group"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/image.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/image.json
deleted file mode 100644
index c820065..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/image.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "image",
- "extends": {
- "$ref": "../media_link.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "image"
- },
- "fullImage": {
- "type": "string"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/issue.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/issue.json
deleted file mode 100644
index 394d571..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/issue.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "issue",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "issue"
- },
- "types": {
- "type": "array",
- "items": {
- "type": "string"
- }
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/job.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/job.json
deleted file mode 100644
index bd2d67f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/job.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "job",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "job"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/list.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/list.json
deleted file mode 100644
index 6fbdd94..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/list.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "list",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "list"
- },
- "items": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "$ref": "../object.json"
- }
- }
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/note.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/note.json
deleted file mode 100644
index b2bab1a..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/note.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "note",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "note"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/offer.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/offer.json
deleted file mode 100644
index f10b529..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/offer.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "offer",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "offer"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/organization.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/organization.json
deleted file mode 100644
index c295af2..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/organization.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "organization",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "organization"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/page.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/page.json
deleted file mode 100644
index 87cad51..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/page.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "page",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "page"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/permission.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/permission.json
deleted file mode 100644
index ac8766a..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/permission.json
+++ /dev/null
@@ -1,36 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "permission",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "permission"
- },
- "scope": {
- "type": "object",
- "extends": {
- "$ref": "../object.json"
- }
- },
- "actions": {
- "type": "array",
- "items": {
- "type": "string",
- "enumeration": [
- "create",
- "delete",
- "modify"
- ]
- }
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/person.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/person.json
deleted file mode 100644
index e508bdd..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/person.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "person",
- "description": "vCard Format. Does not match PoCO",
- "extends": [
- {
- "$ref": "../object.json"
- },
- {
- "$ref": "http://www.json-schema.org/card"
- }
- ],
- "properties": {
- "objectType": {
- "type": "string",
- "default": "person"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/photo-album.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/photo-album.json
deleted file mode 100644
index 15a193c..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/photo-album.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "article",
- "extends": {
- "$ref": "../objectTypes/list.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "photo-album"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/photo.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/photo.json
deleted file mode 100644
index f4e991a..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/photo.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "photo",
- "extends": {
- "$ref": "../objectTypes/file.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "photo"
- },
- "displayName": {
- "type": "string",
- "default": "a photo"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/place.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/place.json
deleted file mode 100644
index 1f5cba6..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/place.json
+++ /dev/null
@@ -1,39 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "place",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "place"
- },
- "address": {
- "type": "object",
- "extends": {
- "$ref": "http://www.json-schema.org/address"
- }
- },
- "position": {
- "type": "object",
- "properties": {
- "altitude": {
- "type": "number"
- },
- "latitude": {
- "type": "number"
- },
- "longitude": {
- "type": "number"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/playlist.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/playlist.json
deleted file mode 100644
index 2ac6ec1..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/playlist.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "playlist",
- "extends": {
- "$ref": "../objectTypes/list.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "playlist"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/process.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/process.json
deleted file mode 100644
index 6c596ee..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/process.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "process",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "process"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/product.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/product.json
deleted file mode 100644
index 2606429..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/product.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "product",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "product"
- },
- "fullImage": {
- "type": "object",
- "extends": {
- "$ref": "../media_link.json"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/property.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/property.json
deleted file mode 100644
index 50e36f3..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/property.json
+++ /dev/null
@@ -1,48 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "property",
- "description": "A property describes name, path and value. Can be used with delete, update or post verbs",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "property"
- },
- "displayName": {
- "type": "string",
- "description": "The human readable name of the property in the appropriate language",
- "optional": true
- },
- "path": {
- "type": "string",
- "description": "dot delimited path to the property in the target. Ex: streetAddress"
- }
- },
- "example": {
- "actor": {
- "id": 1212,
- "displayName": "Peter"
- },
- "verb": "update",
- "time": "2010-08-02T15:29:00Z",
- "object": {
- "objectType": "property",
- "displayName": "street address",
- "path": "streetAddress",
- "content": "234 Amazing St"
- },
- "target": {
- "id": 12121,
- "time": "2010-08-02T15:29:00Z",
- "displayName": "Peter's House"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/question.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/question.json
deleted file mode 100644
index 7a864a0..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/question.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "question",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "question"
- },
- "options": {
- "type": "array",
- "items": {
- "type": "object",
- "extends": {
- "$ref": "../object.json"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/review.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/review.json
deleted file mode 100644
index 8731fde..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/review.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "review",
- "extends": {
- "$ref": "../objectTypes/article.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "review"
- },
- "rating": {
- "type": "number"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/role.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/role.json
deleted file mode 100644
index ae53c34..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/role.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "role",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "role"
- },
- "members": {
- "type": "object",
- "extends": {
- "$ref": "../collection.json"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/service.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/service.json
deleted file mode 100644
index 979695c..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/service.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "service",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "service"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/song.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/song.json
deleted file mode 100644
index 289e5a4..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/song.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "song",
- "extends": {
- "$ref": "../objectTypes/audio.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "song"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/status.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/status.json
deleted file mode 100644
index 1a58923..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/status.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "status",
- "extends": {
- "$ref": "../objectTypes/note.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "song"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/task.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/task.json
deleted file mode 100644
index 608514b..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/task.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "task",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "task"
- },
- "by": {
- "type": "string",
- "format": "date-time"
- },
- "prerequisites": {
- "type": "array",
- "items": {
- "type": "object",
- "$ref": "../activity.json"
- }
- },
- "required": {
- "type": "boolean"
- },
- "supersedes": {
- "type": "array",
- "items": {
- "type": "object",
- "$ref": "../activity.json"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/team.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/team.json
deleted file mode 100644
index bcc6218..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/team.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "team",
- "extends": {
- "$ref": "../object.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "team"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/video.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/video.json
deleted file mode 100644
index 3129a84..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/objectTypes/video.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "video",
- "extends": {
- "$ref": "../objectTypes/file.json"
- },
- "properties": {
- "objectType": {
- "type": "string",
- "default": "video"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/accept.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/accept.json
deleted file mode 100644
index 579d128..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/accept.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Accept",
- "description": "Indicates that that the actor has accepted the object. For instance, a person accepting an award, or accepting an assignment.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "accept"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} accepted {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/access.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/access.json
deleted file mode 100644
index 2131fc5..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/access.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Access",
- "description": "Indicates that the actor has accessed the object. For instance, a person accessing a room, or accessing a file.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "access"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} accessed {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/acknowledge.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/acknowledge.json
deleted file mode 100644
index d071e63..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/acknowledge.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Acknowledge",
- "description": "Indicates that the actor has acknowledged the object. This effectively signals that the actor is aware of the object's existence.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "acknowledge"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} acknowledge {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/add.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/add.json
deleted file mode 100644
index f48715c..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/add.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Add",
- "description": "Indicates that the actor has added the object to the target. For instance, adding a photo to an album.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "add"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} added {object.displayName} to {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/agree.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/agree.json
deleted file mode 100644
index a2c86d5..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/agree.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Agree",
- "description": "Indicates that the actor agrees with the object. For example, a person agreeing with an argument, or expressing agreement with a particular issue.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "agree"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} agrees with {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/append.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/append.json
deleted file mode 100644
index fe46070..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/append.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Append",
- "description": "Indicates that the actor has appended the object to the target. For instance, a person appending a new record to a database.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "append"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} append {object.displayName} to {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/approve.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/approve.json
deleted file mode 100644
index 615d4c3..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/approve.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Approve",
- "description": "Indicates that the actor has approved the object. For instance, a manager might approve a travel request.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "approve"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} approved {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/archive.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/archive.json
deleted file mode 100644
index 1a5455c..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/archive.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Archive",
- "description": "Indicates that the actor has archived the object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "archive"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} archived {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/assign.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/assign.json
deleted file mode 100644
index dd3e5fa..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/assign.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Assign",
- "description": "Indicates that the actor has assigned the object to the target.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "assign"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} assigned {object.displayName} to {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/at.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/at.json
deleted file mode 100644
index 649bd28..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/at.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "At",
- "description": "Indicates that the actor is currently located at the object. For instance, a person being at a specific physical location.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "assign"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} assigned {object.displayName} to {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/attach.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/attach.json
deleted file mode 100644
index efc04de..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/attach.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Attach",
- "description": "Indicates that the actor has attached the object to the target.For instance, a person attaching a file to a wiki page or an email.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "attach"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} attached {object.displayName} to {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/attend.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/attend.json
deleted file mode 100644
index 7b2f32f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/attend.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Attend",
- "description": "Indicates that the actor has attended the object. For instance, a person attending a meeting.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "attend"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} attended {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/author.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/author.json
deleted file mode 100644
index 60616e5..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/author.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Author",
- "description": "Indicates that the actor has authored the object. Note that this is a more specific form of the verb \"create\".",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "author"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} authored {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/authorize.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/authorize.json
deleted file mode 100644
index 13ebe60..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/authorize.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Authorize",
- "description": "Indicates that the actor has authorized the object. If a target is specified, it means that the authorization is specifically in regards to the target. For instance, a service can authorize a person to access a given application; in which case the actor is the service, the object is the person, and the target is the application. In contrast, a person can authorize a request; in which case the actor is the person and the object is the request and there might be no expl [...]
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "authorize"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} authorized {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/borrow.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/borrow.json
deleted file mode 100644
index 59e437e..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/borrow.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Borrow",
- "description": "Indicates that the actor has borrowed the object. If a target is specified, it identifies the entity from which the object was borrowed. For instance, if a person borrows a book from a library, the person is the actor, the book is the object and the library is the target.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "borrow"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} borrow {object.displayName} from {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/build.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/build.json
deleted file mode 100644
index 9e6279d..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/build.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Build",
- "description": "Indicates that the actor has built the object. For example, if a person builds a model or compiles code.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "build"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} built {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/cancel.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/cancel.json
deleted file mode 100644
index 31290cd..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/cancel.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Cancel",
- "description": "Indicates that the actor has canceled the object. For instance, canceling a calendar event.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "cancel"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} cancelled {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/checkin.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/checkin.json
deleted file mode 100644
index f4d1c8f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/checkin.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "checkin",
- "description": "Indicates that the actor has checked-in to the object. For instance, a person checking-in to a Place.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "checkin"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} checked in at {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/close.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/close.json
deleted file mode 100644
index 7726572..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/close.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Close",
- "description": "Indicates that the actor has closed the object. For instance, the object could represent a ticket being tracked in an issue management system.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "close"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} closed {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/complete.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/complete.json
deleted file mode 100644
index a360f93..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/complete.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Complete",
- "description": "Indicates that the actor has completed the object",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "complete"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} completed {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/confirm.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/confirm.json
deleted file mode 100644
index 28975d9..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/confirm.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Confirm",
- "description": "Indicates that the actor has confirmed or agrees with the object. For instance, a software developer might confirm an issue reported against a product.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "confirm"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} confirmed {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/consume.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/consume.json
deleted file mode 100644
index 89890af..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/consume.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Consume",
- "description": "Indicates that the actor has consumed the object. The specific meaning is dependent largely on the object's type. For instance, an actor may \"consume\" an audio object, indicating that the actor has listened to it; or an actor may \"consume\" a book, indicating that the book has been read. As such, the \"consume\" verb is a more generic form of other more specific verbs such as \"read\" and \"play\".",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "consume"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} consumed {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/create.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/create.json
deleted file mode 100644
index f7b7e4d..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/create.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Create",
- "description": "Indicates that the actor has created the object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "create"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} created {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/delete.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/delete.json
deleted file mode 100644
index 6bbf2ae..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/delete.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Delete",
- "description": "Indicates that the actor has deleted the object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "delete"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} deleted {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/deliver.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/deliver.json
deleted file mode 100644
index d0266fd..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/deliver.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Deliver",
- "description": "Indicates that the actor has delivered the object. For example, delivering a package.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "deliver"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} delivered {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/deny.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/deny.json
deleted file mode 100644
index 20b0255..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/deny.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Deny",
- "description": "Indicates that the actor has denied the object. For example, a manager may deny a travel request.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "deny"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} denied {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/disagree.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/disagree.json
deleted file mode 100644
index 84a006a..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/disagree.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Disagree",
- "description": "Indicates that the actor disagrees with the object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "disagree"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} disagrees with {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/dislike.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/dislike.json
deleted file mode 100644
index dc7859d..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/dislike.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Dislike",
- "description": "Indicates that the actor dislikes the object. Note that the \"dislike\" verb is distinct from the \"unlike\" verb which assumes that the object had been previously \"liked\".",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "dislike"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} dislikes {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/experience.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/experience.json
deleted file mode 100644
index 0423cc0..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/experience.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Experience",
- "description": "Indicates that the actor has experienced the object in some manner. Note that, depending on the specific object types used for both the actor and object, the meaning of this verb can overlap that of the \"consume\" and \"play\" verbs. For instance, a person might \"experience\" a movie; or \"play\" the movie; or \"consume\" the movie. The \"experience\" verb can be considered a more generic form of other more specific verbs as \"consume\", \"play\", \"watch\", \"liste [...]
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "experience"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} experienced {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/favorite.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/favorite.json
deleted file mode 100644
index 96d2d16..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/favorite.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Favorite",
- "description": "Indicates that the actor marked the object as an item of special interest.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "favorite"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} marked {object.displayName} as a favorite"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/find.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/find.json
deleted file mode 100644
index 341162f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/find.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Find",
- "description": "Indicates that the actor has found the object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "find"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} found {object.displayName}."
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/flag-as-inappropriate.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/flag-as-inappropriate.json
deleted file mode 100644
index dfb0233..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/flag-as-inappropriate.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Flag-As-Inappropriate",
- "description": "Indicates that the actor has flagged the object as being inappropriate for some reason. When using this verb, the context property can be used to provide additional detail about why the object has been flagged.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "flag-as-inappropriate"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} flagged {object.displayName} as inappropriate."
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/follow.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/follow.json
deleted file mode 100644
index 6e68039..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/follow.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Follow",
- "description": "Indicates that the actor began following the activity of the object. In most cases, the objectType will be a \"person\", but it can potentially be of any type that can sensibly generate activity. Processors MAY ignore (silently drop) successive identical \"follow\" activities.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "follow"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} started following {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/give.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/give.json
deleted file mode 100644
index 93da44c..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/give.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Give",
- "description": "Indicates that the actor is giving an object to the target. Examples include one person giving a badge object to another person. The object identifies the object being given. The target identifies the receiver.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "give"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} gave {object.displayName} to {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/host.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/host.json
deleted file mode 100644
index 165a8d5..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/host.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Host",
- "description": "Indicates that the actor is hosting the object. As in hosting an event, or hosting a service.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "host"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} is hosting {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/ignore.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/ignore.json
deleted file mode 100644
index f023e7a..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/ignore.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Ignore",
- "description": "Indicates that the actor has ignored the object. For instance, this verb may be used when an actor has ignored a friend request, in which case the object may be the request-friend activity.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "ignore"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} ignored {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/insert.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/insert.json
deleted file mode 100644
index 407bdaa..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/insert.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Insert",
- "description": "Indicates that the actor has inserted the object into the target.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "insert"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} inserted {object.displayName} into {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/install.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/install.json
deleted file mode 100644
index 01827e3..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/install.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Install",
- "description": "Indicates that the actor has installed the object, as in installing an application.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "install"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} installed {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/interact.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/interact.json
deleted file mode 100644
index 2e74e40..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/interact.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Interact",
- "description": "Indicates that the actor has interacted with the object. For instance, when one person interacts with another.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "interact"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} interacted with {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/invite.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/invite.json
deleted file mode 100644
index 825920e..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/invite.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Invite",
- "description": "Indicates that the actor has invited the object, typically a person object, to join or participate in the object described by the target. The target could, for instance, be an event, group or a service.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "invite"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} invited {object.displayName} to {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/join.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/join.json
deleted file mode 100644
index 347bb41..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/join.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Join",
- "description": "Indicates that the actor has become a member of the object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "join"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} joined {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/leave.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/leave.json
deleted file mode 100644
index 78311fb..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/leave.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Leave",
- "description": "Indicates that the actor has left the object. For instance, a Person leaving a Group or checking-out of a Place.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "leave"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} left {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/like.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/like.json
deleted file mode 100644
index 056bd0e..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/like.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Like",
- "description": "Indicates that the actor marked the object as an item of special interest.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "like"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} likes {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/listen.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/listen.json
deleted file mode 100644
index 121e3c9..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/listen.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Listen",
- "description": "Indicates that the actor has listened to the object. This is typically only applicable for objects representing audio content, such as music, an audio-book, or a radio broadcast.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "listen"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} listened to {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/lose.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/lose.json
deleted file mode 100644
index 7b36c11..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/lose.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Lose",
- "description": "Indicates that the actor has lost the object. For instance, if a person loses a game.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "lose"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} lost {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/make-friend.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/make-friend.json
deleted file mode 100644
index ec9fca9..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/make-friend.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Befriend",
- "description": "Indicates the creation of a friendship that is reciprocated by the object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "make-friend"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} is now friends with {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/open.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/open.json
deleted file mode 100644
index 77945aa..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/open.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Open",
- "description": "Indicates that the actor has opened the object. For instance, the object could represent a ticket being tracked in an issue management system.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "open"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} opened {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/play.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/play.json
deleted file mode 100644
index f4c9c6a..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/play.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Play",
- "description": "Indicates that the actor spent some time enjoying the object. For example, if the object is a video this indicates that the subject watched all or part of the video.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "play"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} played {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/post.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/post.json
deleted file mode 100644
index 44f9f6c..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/post.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Post",
- "description": "To publish an object",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "post"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} posted {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/present.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/present.json
deleted file mode 100644
index 266d228..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/present.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Present",
- "description": "Indicates that the actor has presented the object. For instance, when a person gives a presentation at a conference.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "present"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} presented {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/purchase.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/purchase.json
deleted file mode 100644
index 3d68273..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/purchase.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Purchase",
- "description": "Indicates that the actor has purchased the object. If a target is specified, in indicates the entity from which the object was purchased.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "purchase"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} purchased {object.displayName} from {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/qualify.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/qualify.json
deleted file mode 100644
index b23b851..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/qualify.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Qualify",
- "description": "Indicates that the actor has qualified for the object. If a target is specified, it indicates the context within which the qualification applies.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "qualify"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} qualified {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/read.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/read.json
deleted file mode 100644
index 1e2b30a..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/read.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Read",
- "description": "Indicates that the actor read the object. This is typically only applicable for objects representing printed or written content, such as a book, a message or a comment.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "read"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} read {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/receive.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/receive.json
deleted file mode 100644
index b636d97..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/receive.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Receive",
- "description": "Indicates that the actor is receiving an object. Examples include a person receiving a badge object. The object identifies the object being received.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "receive"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} received {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/reject.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/reject.json
deleted file mode 100644
index dae5409..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/reject.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Reject",
- "description": "Indicates that the actor has rejected the object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "reject"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} rejected {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/remove-friend.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/remove-friend.json
deleted file mode 100644
index 7eaff1c..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/remove-friend.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "UnFriend",
- "description": "Indicates that the actor has removed the object from the collection of friends.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "remove-friend"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} removed friend {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/remove.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/remove.json
deleted file mode 100644
index affbb3c..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/remove.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Remove",
- "description": "Indicates that the actor has removed the object from the target.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "remove"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} removed {object.displayName} from {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/replace.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/replace.json
deleted file mode 100644
index 2df0da5..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/replace.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Replace",
- "description": "Indicates that the actor has replaced the target with the object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "replace"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} replaced {target.displayName} with {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/request-friend.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/request-friend.json
deleted file mode 100644
index fd6b870..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/request-friend.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "RequestFriend",
- "description": "Indicates the creation of a friendship that has not yet been reciprocated by the object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "request-friend"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} sent a friend request to {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/request.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/request.json
deleted file mode 100644
index e03f0fe..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/request.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Request",
- "description": "Indicates that the actor has requested the object. If a target is specified, it indicates the entity from which the object is being requested.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "request"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} requested {object.displayName} from {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/resolve.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/resolve.json
deleted file mode 100644
index db93a36..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/resolve.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Resolve",
- "description": "Indicates that the actor has resolved the object. For instance, the object could represent a ticket being tracked in an issue management system.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "resolve"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} resolved {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/retract.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/retract.json
deleted file mode 100644
index c41d7ad..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/retract.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Retract",
- "description": "Indicates that the actor has retracted the object. For instance, if an actor wishes to retract a previously published activity, the object would be the previously published activity that is being retracted.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "retract"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} retracted {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/return.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/return.json
deleted file mode 100644
index 77cb31e..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/return.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Return",
- "description": "Indicates that the actor has returned the object. If a target is specified, it indicates the entity to which the object was returned.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "return"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} returned {object.displayName} to {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/rsvp-maybe.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/rsvp-maybe.json
deleted file mode 100644
index c7219d1..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/rsvp-maybe.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Rsvp Maybe",
- "description": "To indicate that the actor may attend an event",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "rsvp-maybe"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} might attend {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/rsvp-no.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/rsvp-no.json
deleted file mode 100644
index c06537f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/rsvp-no.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Rsvp No",
- "description": "To indicate that the actor will not attend an event",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "rsvp-no"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} will not attend {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/rsvp-yes.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/rsvp-yes.json
deleted file mode 100644
index f1a57a1..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/rsvp-yes.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Rsvp Yes",
- "description": "To indicate that the actor will attend an event",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "rsvp-yes"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} is attending {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/satisfy.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/satisfy.json
deleted file mode 100644
index 8d141e1..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/satisfy.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Satisfy",
- "description": "Indicates that the actor has satisfied the object. If a target is specified, it indicate the context within which the object was satisfied. For instance, if a person satisfies the requirements for a particular challenge, the person is the actor; the requirement is the object; and the challenge is the target.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "satisfy"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} has satisfied {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/save.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/save.json
deleted file mode 100644
index 9d22cb9..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/save.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Save",
- "description": "Indicates that the actor has called out the object as being of interest primarily to him- or herself. Though this action MAY be shared publicly, the implication is that the object has been saved primarily for the actor's own benefit rather than to show it to others as would be indicated by the \"share\" verb.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "save"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} saved {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/schedule.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/schedule.json
deleted file mode 100644
index dd97a1d..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/schedule.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Schedule",
- "description": "Indicates that the actor has scheduled the object. For instance, scheduling a meeting.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "schedule"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} scheduled {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/search.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/search.json
deleted file mode 100644
index a88296a..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/search.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Search",
- "description": "Indicates that the actor is or has searched for the object. If a target is specified, it indicates the context within which the search is or has been conducted.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "search"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} search for {object.displayName} in {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/sell.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/sell.json
deleted file mode 100644
index 1863c8f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/sell.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Sell",
- "description": "Indicates that the actor has sold the object. If a target is specified, it indicates the entity to which the object was sold.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "sell"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} sold {object.displayName} to {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/send.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/send.json
deleted file mode 100644
index 915e32c..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/send.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Send",
- "description": "Indicates that the actor has sent the object to the target.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "send"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} sent {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/share.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/share.json
deleted file mode 100644
index 6452858..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/share.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Share",
- "description": "To share an object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "share"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} shared {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/sponsor.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/sponsor.json
deleted file mode 100644
index 0ed2a75..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/sponsor.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Sponsor",
- "description": "Indicates that the actor has sponsored the object. If a target is specified, it indicates the context within which the sponsorship is offered. For instance, a company can sponsor an event; or an individual can sponsor a project; etc.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "sponsor"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} sponsored {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/start.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/start.json
deleted file mode 100644
index 64c5321..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/start.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Start",
- "description": "Indicates that the actor has started the object. For instance, when a person starts a project.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "start"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} started {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/stop-following.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/stop-following.json
deleted file mode 100644
index da3e607..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/stop-following.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "UnFollow",
- "description": "Indicates that the actor has stopped following the object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "stop-following"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} stopped following {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/submit.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/submit.json
deleted file mode 100644
index 66d7f8d..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/submit.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Submit",
- "description": "Indicates that the actor has submitted the object. If a target is specified, it indicates the entity to which the object was submitted.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "submit"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} submitted {object.displayName} to {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/tag.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/tag.json
deleted file mode 100644
index b2bda80..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/tag.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Tag",
- "description": "Indicates that the actor has associated the object with the target. For example, if the actor specifies that a particular user appears in a photo. the object is the user and the target is the photo.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "tag"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} tagged {object.displayName} as {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/terminate.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/terminate.json
deleted file mode 100644
index 10cfc12..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/terminate.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Terminate",
- "description": "Indicates that the actor has terminated the object.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "terminate"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} terminated {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/tie.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/tie.json
deleted file mode 100644
index 2346a8b..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/tie.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Tie",
- "description": "Indicates that the actor has neither won or lost the object. This verb is generally only applicable when the object represents some form of competition, such as a game.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "tie"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} tied at {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unfavorite.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unfavorite.json
deleted file mode 100644
index d5b2546..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unfavorite.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "UnFavorite",
- "description": "Indicates that the actor has removed the object from the collection of favorited items.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "unfavorite"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} removed {object.displayName} as a favorite"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unlike.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unlike.json
deleted file mode 100644
index 2533378..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unlike.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "UnLike",
- "description": "Indicates that the actor has removed the object from the collection of liked items.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "unlike"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} no longer likes {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unsatisfy.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unsatisfy.json
deleted file mode 100644
index 51fb82b..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unsatisfy.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "UnSatisfy",
- "description": "Indicates that the actor has not satisfied the object. If a target is specified, it indicates the context within which the object was not satisfied. For instance, if a person fails to satisfy the requirements of some particular challenge, the person is the actor; the requirement is the object and the challenge is the target.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "unsatisfy"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} has not satisfied {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unsave.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unsave.json
deleted file mode 100644
index d5086cb..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unsave.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "UnSave",
- "description": "Indicates that the actor has removed the object from the collection of saved items.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "unsave"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} unsaved {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unshare.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unshare.json
deleted file mode 100644
index a2b853d..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/unshare.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "UnSave",
- "description": "Indicates that the actor is no longer sharing the object. If a target is specified, it indicates the entity with whom the object is no longer being shared.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "unshare"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} is no longer sharing {object.displayName} with {target.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/update.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/update.json
deleted file mode 100644
index bba9368..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/update.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Update",
- "description": "The \"update\" verb indicates that the actor has modified the object. Use of the \"update\" verb is generally reserved to indicate modifications to existing objects or data such as changing an existing user's profile information.",
- "notes": "The new property value can be a scalar value or an object. See the property object type",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "update"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} updated {target.displayName}'s {object.displayName} to {object.value}"
- },
- "object": {
- "type": "object",
- "$ref": "../object.json"
- },
- "target": {
- "type": "object",
- "optional": true,
- "$ref": "../object.json"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/use.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/use.json
deleted file mode 100644
index 69201f2..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/use.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Use",
- "description": "Indicates that the actor has used the object in some manner.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "use"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} used {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/watch.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/watch.json
deleted file mode 100644
index b8c4f34..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/watch.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Watch",
- "description": "Indicates that the actor has watched the object. This verb is typically applicable only when the object represents dynamic, visible content such as a movie, a television show or a public performance.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "watch"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} watched {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/win.json b/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/win.json
deleted file mode 100644
index 57ef70b..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/jsonschema/verbs/win.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-03/schema",
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "id":"#",
- "type": "object",
- "title": "Win",
- "description": "Indicates that the actor has won the object. For instance, if a person wins a game.",
- "extends": {
- "$ref": "../activity.json"
- },
- "properties": {
- "verb": {
- "type": "string",
- "default": "win"
- },
- "title": {
- "type": "string",
- "default": "{actor.displayName} won {object.displayName}"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/main/xmlschema/activity.xsd b/streams-schemas/streams-schema-activitystreams/src/main/xmlschema/activity.xsd
deleted file mode 100644
index 1d81b44..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/main/xmlschema/activity.xsd
+++ /dev/null
@@ -1,94 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<xs:schema version="1.0" xmlns:xs="http://www.w3.org/2001/XMLSchema">
-
- <xs:element name="id" type="xs:string"/>
- <xs:element name="actor" type="objectType"/>
- <xs:element name="verb" type="verbType"/>
- <xs:element name="object" type="objectType"/>
- <xs:element name="target" type="objectType"/>
- <xs:element name="object-type" type="xs:string"/>
- <xs:element name="published" type="xs:dateTime"/>
- <xs:element name="updated" type="xs:dateTime"/>
- <xs:element name="provider" type="objectType"/>
- <xs:element name="title" type="xs:string"/>
- <xs:element name="content" type="xs:string"/>
- <xs:element name="url" type="xs:string"/>
-
- <xs:simpleType name="verbType">
- <xs:restriction base="xs:string">
- <xs:enumeration value="favorite"/>
- <xs:enumeration value="follow"/>
- <xs:enumeration value="join"/>
- <xs:enumeration value="like"/>
- <xs:enumeration value="make-friend"/>
- <xs:enumeration value="play"/>
- <xs:enumeration value="post"/>
- <xs:enumeration value="rsvp-yes"/>
- <xs:enumeration value="rsvp-no"/>
- <xs:enumeration value="rsvp-maybe"/>
- <xs:enumeration value="save"/>
- <xs:enumeration value="share"/>
- <xs:enumeration value="tag"/>
- <xs:enumeration value="update"/>
- </xs:restriction>
- </xs:simpleType>
-
- <xs:complexType name="objectType">
- <xs:sequence>
- <xs:element name="id" type="xs:string"/>
- <xs:element name="image" type="mediaLink"/>
- <xs:element name="displayName" type="xs:string"/>
- <xs:element name="summary" type="xs:string"/>
- <xs:element name="content" type="xs:string"/>
- <xs:element name="url" type="xs:string"/>
- <xs:element name="published" type="xs:string"/>
- <xs:element name="updated" type="xs:string"/>
- <xs:element name="object-type" >
- <xs:simpleType>
- <xs:restriction base="xs:string">
- <xs:enumeration value="article"/>
- <xs:enumeration value="audio"/>
- <xs:enumeration value="badge"/>
- <xs:enumeration value="collection"/>
- <xs:enumeration value="comment"/>
- <xs:enumeration value="course"/>
- <xs:enumeration value="file"/>
- <xs:enumeration value="image"/>
- <xs:enumeration value="person"/>
- <xs:enumeration value="review"/>
- <xs:enumeration value="service"/>
- </xs:restriction>
- </xs:simpleType>
- </xs:element>
- </xs:sequence>
-
- </xs:complexType>
-
- <xs:complexType name="mediaLink">
- <xs:sequence>
- <xs:element name="duration" type="xs:positiveInteger"/>
- <xs:element name="height" type="xs:positiveInteger"/>
- <xs:element name="width" type="xs:positiveInteger"/>
- <xs:element name="url" type="xs:string"/>
- </xs:sequence>
- </xs:complexType>
-
-</xs:schema>
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/java/org/w3c/activitystreams/test/SchemaValidationTest.java b/streams-schemas/streams-schema-activitystreams/src/test/java/org/w3c/activitystreams/test/SchemaValidationTest.java
deleted file mode 100644
index a06c2f5..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/java/org/w3c/activitystreams/test/SchemaValidationTest.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.w3c.activitystreams.test;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.networknt.schema.JsonSchema;
-import com.networknt.schema.JsonSchemaFactory;
-import com.networknt.schema.ValidationMessage;
-import org.apache.commons.io.IOUtils;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.Set;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.Is.is;
-
-/**
- * Test validity of documents vs schemas.
- */
-public class SchemaValidationTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SchemaValidationTest.class);
-
- private static final ObjectMapper MAPPER = new ObjectMapper();
-
- /**
- * Tests that activities matching core-ex* can be parsed by apache streams.
- *
- * @throws Exception Test Exception
- */
- @Test
- public void testValidateToSchema() throws Exception {
-
- JsonSchemaFactory factory = new JsonSchemaFactory();
-
- InputStream testActivityFolderStream = SchemaValidationTest.class.getClassLoader()
- .getResourceAsStream("activities");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- for (String file : files) {
- if ( !file.startsWith(".") ) {
-
- LOGGER.info("Test File: activities/" + file);
- String testFileString = new String(Files.readAllBytes(Paths.get("target/test-classes/activities/" + file)));
- LOGGER.info("Test Document JSON: " + testFileString);
- JsonNode testNode = MAPPER.readValue(testFileString, ObjectNode.class);
- LOGGER.info("Test Document Object:" + testNode);
- LOGGER.info("Test Schema File: " + "target/classes/verbs/" + file);
- String testSchemaString = new String(Files.readAllBytes(Paths.get("target/classes/verbs/" + file)));
- LOGGER.info("Test Schema JSON: " + testSchemaString);
- JsonNode testSchemaNode = MAPPER.readValue(testFileString, ObjectNode.class);
- LOGGER.info("Test Schema Object:" + testSchemaNode);
- JsonSchema testSchema = factory.getSchema(testSchemaNode);
- LOGGER.info("Test Schema:" + testSchema);
-
- Set<ValidationMessage> errors = testSchema.validate(testNode);
- assertThat(errors.size(), is(0));
-
- }
- }
- }
-
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/accept.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/accept.json
deleted file mode 100644
index c1dfd5f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/accept.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "accept",
- "object": {
- "objectType": "job",
- "displayName": "Director of Marketing"
- },
- "title": "Sally accepted the Director of Marketing job."
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/access.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/access.json
deleted file mode 100644
index 93bc6e2..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/access.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "access",
- "object": {
- "objectType": "file",
- "displayName": "4Q2012 Sales Forecast.xls"
- },
- "published": "2012-12-12T12:12:12Z",
- "title": "Joe accessed the file \"4Q2012 Sales Forecast.xls\""
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/acknowledge.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/acknowledge.json
deleted file mode 100644
index 2b69fa7..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/acknowledge.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "acknowledge",
- "object": {
- "objectType": "issue",
- "displayName": "#123: There is a problem with the build"
- },
- "content": "Sally acknowledged Issue #123"
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/add.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/add.json
deleted file mode 100644
index b4e8151..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/add.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "add",
- "object": {
- "objectType": "image",
- "displayName": "My cat",
- "fullImage": {"url": "http://example.org/cat.jpg"}
- },
- "target": {
- "objectType": "collection",
- "displayName": "Joe's Photo Album",
- "objectTypes": ["image"]
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/agree.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/agree.json
deleted file mode 100644
index 3034cd8..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/agree.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "agree",
- "object": {
- "objectType": "article",
- "displayName": "Some Random Article Online"
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/append.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/append.json
deleted file mode 100644
index d8fb3be..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/append.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "append",
- "object": {"content": "This is some text"},
- "target": {
- "objectType": "file",
- "displayName": "log.txt"
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/approve.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/approve.json
deleted file mode 100644
index b6f378e..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/approve.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "approve",
- "object": {
- "objectType": "task",
- "actor": {"displayName": "Joe"},
- "verb": "join",
- "object": {
- "objectType": "group",
- "displayName": "Administrators"
- }
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/archive.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/archive.json
deleted file mode 100644
index 557dcdf..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/archive.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "archive",
- "object": {
- "objectType": "file",
- "displayName": "4Q2012 Sales Forecast.xls"
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/assign.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/assign.json
deleted file mode 100644
index 51b8a51..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/assign.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "assign",
- "object": {
- "objectType": "issue",
- "displayName": "Issue #123: Some Issue"
- },
- "target": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "title": "Sally assigned Issue #123 to Joe"
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/at.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/at.json
deleted file mode 100644
index 64a77f9..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/at.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "at",
- "object": {
- "objectType": "place",
- "displayName": "Acme, Co."
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/attach.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/attach.json
deleted file mode 100644
index 716616b..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/attach.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "attach",
- "object": {
- "objectType": "binary",
- "data": "dGhpcyBpcyB1bmNvbXByZXNzZWQgZGF0YQo="
- },
- "target": {
- "objectType": "issue",
- "displayName": "Issue #123"
- },
- "title": "Sally added an attachment to Issue #123"
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/attend.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/attend.json
deleted file mode 100644
index f8c0838..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/attend.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "attend",
- "object": {
- "objectType": "event",
- "displayName": "Sally's Meeting"
- }
-}
\ No newline at end of file
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/author.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/author.json
deleted file mode 100644
index 1903700..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/author.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "author",
- "object": {
- "objectType": "file",
- "displayName": "4Q2012 Sales Forecast.xls"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/authorize.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/authorize.json
deleted file mode 100644
index 2d9d1b3..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/authorize.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "authorize",
- "object": {
- "objectType": "task",
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "access",
- "object": {
- "objectType": "place",
- "displayName": "Joe's Home"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/borrow.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/borrow.json
deleted file mode 100644
index e21809a..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/borrow.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "borrow",
- "object": {
- "objectType": "book",
- "displayName": "Cloud Atlas"
- },
- "target": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "title": "Sally borrowed the book 'Cloud Atlas' from Joe"
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/build.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/build.json
deleted file mode 100644
index 78878e1..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/build.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "build",
- "object": {
- "objectType": "application",
- "displayName": "MyApp Builder 12345"
- }
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/cancel.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/cancel.json
deleted file mode 100644
index b7aba81..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/cancel.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "cancel",
- "object": {
- "objectType": "offer",
- "displayName": "Free Money!"
- },
- "title": "Sally cancelled the offer for free money."
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/checkin.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/checkin.json
deleted file mode 100644
index 97216b9..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/checkin.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "checkin",
- "object": {
- "objectType": "place",
- "displayName": "Acme, Co"
- },
- "title": "Joe checked in at Acme, Co"
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/close.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/close.json
deleted file mode 100644
index 362e3f6..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/close.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "close",
- "object": {
- "objectType": "issue",
- "displayName": "Issue #123"
- },
- "title": "Joe closed issue #123"
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/complete.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/complete.json
deleted file mode 100644
index 06694a7..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/complete.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "complete",
- "object": {
- "objectType": "process",
- "displayName": "Some long process"
- }
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/confirm.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/confirm.json
deleted file mode 100644
index 9307c38..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/confirm.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "confirm",
- "object": {
- "objectType": "issue",
- "displayName": "Issue #123"
- },
- "title": "Joe confirmed issue #123"
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/consume.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/consume.json
deleted file mode 100644
index ed907e3..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/consume.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "consume",
- "object": {
- "objectType": "product",
- "displayName": "Some amazing product"
- }
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/create.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/create.json
deleted file mode 100644
index 5d8afb3..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/create.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "create",
- "object": {
- "objectType": "product",
- "displayName": "Some amazing product"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/delete.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/delete.json
deleted file mode 100644
index 6943046..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/delete.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "delete",
- "object": {
- "objectType": "file",
- "displayName": "4Q2012 Sales Forecast.xls"
- },
- "title": "Joe is probably going to get fired."
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/deliver.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/deliver.json
deleted file mode 100644
index 620f0b5..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/deliver.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "deliver",
- "object": {
- "objectType": "note",
- "displayName": "Bad News",
- "content": "Joe deleted the sales forecast"
- },
- "target": {
- "objectType": "person",
- "displayName": "Joe's Boss"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/deny.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/deny.json
deleted file mode 100644
index b83ca60..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/deny.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "deny",
- "object": {
- "objectType": "task",
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "delete",
- "object": {
- "objectType": "file",
- "displayName": "4Q2012 Sales Forecast.xls"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/disagree.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/disagree.json
deleted file mode 100644
index 8614e88..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/disagree.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "disagree",
- "object": {
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "deny",
- "object": {
- "objectType": "task",
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "delete",
- "object": {
- "objectType": "file",
- "displayName": "4Q2012 Sales Forecast.xls"
- }
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/dislike.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/dislike.json
deleted file mode 100644
index 16c2148..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/dislike.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "dislike",
- "object": {
- "objectType": "person",
- "displayName": "Sally"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/experience.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/experience.json
deleted file mode 100644
index d81d024..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/experience.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally"
- },
- "verb": "experience",
- "object": {
- "objectType": "event",
- "displayName": "Disciplinary Action for Joe"
- }
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/favorite.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/favorite.json
deleted file mode 100644
index 3df99bf..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/favorite.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe's Boss"
- },
- "verb": "favorite",
- "object": {
- "objectType": "person",
- "displayName": "Sally"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/find.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/find.json
deleted file mode 100644
index 9c25c59..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/find.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "find",
- "object": {
- "objectType": "application",
- "displayName": "Unapproved Software Application"
- },
- "location": {
- "objectType": "place",
- "displayName": "Sally's Computer"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/flag-as-inappropriate.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/flag-as-inappropriate.json
deleted file mode 100644
index 6f4d202..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/flag-as-inappropriate.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Sally's Boss"
- },
- "verb": "flag-as-inappropriate",
- "object": {
- "objectType": "application",
- "displayName": "Unapproved Software Application",
- "location": {
- "displayName": "Sally's Computer"
- }
- },
- "context": {
- "objectType": "issue",
- "displayName": "Issue #125",
- "types": ["http://example.org/violation-of-corporate-policy"]
- }
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/follow.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/follow.json
deleted file mode 100644
index fc8c992..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/follow.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe"
- },
- "verb": "follow",
- "object": {
- "objectType": "issue",
- "displayName": "Issue #125"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/give.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/give.json
deleted file mode 100644
index f145296..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/give.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Joe's Boss"
- },
- "verb": "give",
- "object": {
- "objectType": "note",
- "displayName": "Notice of Employment Termination"
- },
- "target": {
- "objectType": "collection",
- "items": [
- {
- "objectType": "person",
- "displayName": "Joe"
- },
- {
- "objectType": "person",
- "displayName": "Sally"
- }
- ]
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/host.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/host.json
deleted file mode 100644
index 3331353..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/host.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Bob"
- },
- "verb": "host",
- "object": {
- "objectType": "event",
- "displayName": "Job Interview"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/ignore.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/ignore.json
deleted file mode 100644
index 012013a..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/ignore.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "ignore",
- "object": {
- "objectType": "note",
- "displayName": "Joe's request for his job back."
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/insert.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/insert.json
deleted file mode 100644
index 57bda66..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/insert.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Bob"
- },
- "verb": "insert",
- "object": {
- "objectType": "note",
- "displayName": "Notes about Employee Disciplinary Actions"
- },
- "target": {
- "objectType": "file",
- "displayName": "2013 Corporate Policy Updates.doc"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/install.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/install.json
deleted file mode 100644
index 1ecdecd..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/install.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "install",
- "object": {
- "objectType": "application",
- "displayName": "Approved Software Scanning Tool",
- "location": {
- "displayName": "All computers in Building A"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/interact.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/interact.json
deleted file mode 100644
index f427513..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/interact.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Bob"
- },
- "verb": "interact",
- "object": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "title": "Bob called Laura."
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/invite.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/invite.json
deleted file mode 100644
index 7c84b78..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/invite.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "invite",
- "object": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "target": {
- "objectType": "event",
- "displayName": "Job Interview"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/join.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/join.json
deleted file mode 100644
index 7996a9b..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/join.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "join",
- "object": {
- "objectType": "organization",
- "displayName": "Acme, Co"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/leave.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/leave.json
deleted file mode 100644
index 08516c5..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/leave.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "leave",
- "object": {
- "objectType": "organization",
- "displayName": "Other, Co"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/like.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/like.json
deleted file mode 100644
index 853676f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/like.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "like",
- "object": {
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "join",
- "object": {
- "objectType": "organization",
- "displayName": "Acme, Co"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/listen.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/listen.json
deleted file mode 100644
index 1268060..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/listen.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "listen",
- "object": {
- "objectType": "audio",
- "displayName": "Welcome to the Company (Podcast).mp3"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/lose.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/lose.json
deleted file mode 100644
index 073e914..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/lose.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "organization",
- "displayName": "New York Yankees"
- },
- "verb": "lose",
- "object": {
- "objectType": "game",
- "displayName": "World Series"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/make-friend.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/make-friend.json
deleted file mode 100644
index 1acf539..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/make-friend.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "make-friend",
- "object": {
- "objectType": "person",
- "displayName": "Laura"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/open.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/open.json
deleted file mode 100644
index 4bf47cd..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/open.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "open",
- "object": {
- "objectType": "issue",
- "displayName": "Issue #126"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/play.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/play.json
deleted file mode 100644
index 0605662..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/play.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "play",
- "object": {
- "objectType": "audio",
- "displayName": "Call Me Maybe"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/post.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/post.json
deleted file mode 100644
index 5a3471f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/post.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "published": "2011-02-10T15:04:55Z",
- "actor": {
- "url": "http://example.org/martin",
- "objectType" : "person",
- "id": "tag:example.org,2011:martin",
- "image": {
- "url": "http://example.org/martin/image",
- "width": 250,
- "height": 250
- },
- "displayName": "Martin Smith"
- },
- "verb": "post",
- "object" : {
- "url": "http://example.org/blog/2011/02/entry",
- "id": "tag:example.org,2011:abc123/xyz"
- },
- "target" : {
- "url": "http://example.org/blog/",
- "objectType": "blog",
- "id": "tag:example.org,2011:abc123",
- "displayName": "Martin's Blog"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/present.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/present.json
deleted file mode 100644
index 8f4df12..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/present.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "present",
- "object": {
- "objectType": "file",
- "displayName": "1Q2013 Sales Forecast.ppt"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/purchase.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/purchase.json
deleted file mode 100644
index 3354597..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/purchase.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "purchase",
- "object": {
- "objectType": "video",
- "displayName": "The Avengers"
- },
- "title": "Mark purchased the movie, The Avengers"
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/qualify.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/qualify.json
deleted file mode 100644
index 630bda0..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/qualify.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "qualify",
- "object": {
- "objectType": "offer",
- "displayName": "Free Money!"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/read.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/read.json
deleted file mode 100644
index 3509886..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/read.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "read",
- "object": {
- "objectType": "book",
- "displayName": "Cloud Atlas"
- }
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/receive.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/receive.json
deleted file mode 100644
index d61245e..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/receive.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "receive",
- "object": {
- "objectType": "badge",
- "displayName": "Most Checkins in 24 hours"
- },
- "title": "Laura was awarded a badge for \"Most Checkins in 24 hours\""
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/reject.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/reject.json
deleted file mode 100644
index 84d1ab1..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/reject.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "reject",
- "object": {
- "objectType": "issue",
- "displayName": "Issue #126"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/remove-friend.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/remove-friend.json
deleted file mode 100644
index 0a593e9..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/remove-friend.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "remove-friend",
- "object": {
- "objectType": "person",
- "displayName": "Laura"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/remove.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/remove.json
deleted file mode 100644
index 1f386e9..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/remove.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "remove",
- "object": {
- "objectType": "image",
- "displayName": "Cat Photo",
- "fullImage": {
- "url": "http://example.org/cats.jpg"
- }
- },
- "target": {
- "objectType": "collection",
- "displayName": "Cat Photo Album",
- "objectTypes": ["image"]
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/replace.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/replace.json
deleted file mode 100644
index f19a35c..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/replace.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "replace",
- "object": {
- "objectType": "file",
- "displayName": "Updated 1Q2014 Sales Forecast.xls"
- },
- "target": {
- "objectType": "file",
- "displayName": "1Q2014 Sales Forecast.xls"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/request-friend.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/request-friend.json
deleted file mode 100644
index 3e571ee..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/request-friend.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "request-friend",
- "object": {
- "objectType": "person",
- "displayName": "Laura"
- }
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/request.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/request.json
deleted file mode 100644
index 3e7c285..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/request.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "request",
- "object": {
- "objectType": "task",
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "join",
- "object": {
- "objectType": "event",
- "displayName": "The Big Meeting"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/resolve.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/resolve.json
deleted file mode 100644
index 14998cf..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/resolve.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "resolve",
- "object": {
- "objectType": "issue",
- "displayName": "Issue #126"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/retract.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/retract.json
deleted file mode 100644
index 9229868..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/retract.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "retract",
- "object": {
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "return",
- "object": {
- "objectType": "book",
- "displayName": "Cloud Atlas"
- },
- "target": {
- "objectType": "person",
- "displayName": "Laura"
- }
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/return.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/return.json
deleted file mode 100644
index d6b8861..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/return.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "return",
- "object": {
- "objectType": "book",
- "displayName": "Cloud Atlas"
- },
- "target": {
- "objectType": "person",
- "displayName": "Laura"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/rsvp-maybe.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/rsvp-maybe.json
deleted file mode 100644
index b1ec1b5..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/rsvp-maybe.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "rsvp-maybe",
- "object": {
- "objectType": "event",
- "displayName": "The Big Meeting"
- },
- "title": "Laura might attend The Big Meeting"
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/rsvp-no.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/rsvp-no.json
deleted file mode 100644
index 532b096..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/rsvp-no.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "rsvp-no",
- "object": {
- "objectType": "event",
- "displayName": "The Big Meeting"
- },
- "title": "Mark will not attend the Big Meeting"
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/rsvp-yes.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/rsvp-yes.json
deleted file mode 100644
index 170a5c4..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/rsvp-yes.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "rsvp-yes",
- "object": {
- "objectType": "event",
- "displayName": "The Big Meeting"
- },
- "title": "Laura will attend the Big Meeting"
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/satisfy.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/satisfy.json
deleted file mode 100644
index 16151fa..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/satisfy.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "satisfy",
- "object": {
- "objectType": "http://example.org/condition",
- "displayName": "Some Condition"
- },
- "target": {
- "objectType": "http://example.org/parole",
- "displayName": "Terms of Parole"
- },
- "title": "Mark has satisfied a condition of his parole."
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/save.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/save.json
deleted file mode 100644
index 70c0f5f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/save.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "save",
- "object": {
- "objectType": "note",
- "displayName": "A note about something important"
- },
- "target": {
- "objectType": "collection",
- "displayName": "Laura's Reading List"
- },
- "title": "Laura saved the note to her reading list"
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/schedule.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/schedule.json
deleted file mode 100644
index f214290..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/schedule.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "schedule",
- "object": {
- "objectType": "event",
- "displayName": "The Big Meeting"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/search.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/search.json
deleted file mode 100644
index 511b612..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/search.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "search",
- "object": {
- "objectType": "place",
- "displayName": "Big Hotel",
- "address": {
- "locality": "New York",
- "region": "NY"
- }
- },
- "title": "Laura searched for a hotel in New York City, NY"
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/sell.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/sell.json
deleted file mode 100644
index ab07fb0..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/sell.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "sell",
- "object": {
- "objectType": "product",
- "displayName": "A cool product"
- },
- "target": {
- "objectType": "person",
- "displayName": "Laura"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/send.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/send.json
deleted file mode 100644
index b392ce8..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/send.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "send",
- "object": {
- "objectType": "note",
- "content": "Thank you for the cool product."
- },
- "target": {
- "objectType": "person",
- "displayName": "Mark"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/share.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/share.json
deleted file mode 100644
index dc7b9f1..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/share.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "share",
- "object": {
- "objectType": "note",
- "displayName": "An important note"
- },
- "title": "Mark shared an important note."
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/sponsor.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/sponsor.json
deleted file mode 100644
index 156c322..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/sponsor.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "organization",
- "displayName": "Acme, Co"
- },
- "verb": "sponsor",
- "object": {
- "objectType": "game",
- "displayName": "World Series"
- },
- "title": "Acme, Co sponsored the World Series"
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/start.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/start.json
deleted file mode 100644
index b9aede2..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/start.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "start",
- "object": {
- "objectType": "process",
- "displayName": "A Long Running Process"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/stop-following.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/stop-following.json
deleted file mode 100644
index 04b83e7..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/stop-following.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "stop-following",
- "object": {
- "objectType": "person",
- "displayName": "Mark"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/submit.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/submit.json
deleted file mode 100644
index 0fd3c34..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/submit.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "submit",
- "object": {
- "objectType": "issue",
- "displayName": "Issue #127"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/tag.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/tag.json
deleted file mode 100644
index dba6112..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/tag.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "tag",
- "object": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "target": {
- "objectType": "image",
- "displayName": "Pictures of my cats"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/terminate.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/terminate.json
deleted file mode 100644
index f4d886e..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/terminate.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "terminate",
- "object": {
- "objectType": "process",
- "displayName": "A long running process"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/tie.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/tie.json
deleted file mode 100644
index 71a3c39..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/tie.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "collection",
- "items": [
- {
- "objectType": "organization",
- "displayName": "New York Giants"
- },
- {
- "objectType": "organization",
- "displayName": "Oakland Raiders"
- }
- ]
- },
- "verb": "tie",
- "object": {
- "objectType": "game",
- "displayName": "Super Bowl"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unfavorite.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unfavorite.json
deleted file mode 100644
index b1e7bcd..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unfavorite.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "unfavorite",
- "object": {
- "objectType": "article",
- "displayName": "Some article"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unlike.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unlike.json
deleted file mode 100644
index 9bc583f..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unlike.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "unlike",
- "object": {
- "objectType": "article",
- "displayName": "Some article"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unsatisfy.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unsatisfy.json
deleted file mode 100644
index 739823c..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unsatisfy.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "unsatisfy",
- "object": {
- "objectType": "http://example.org/condition",
- "displayName": "Some Condition"
- },
- "target": {
- "objectType": "http://example.org/parole",
- "displayName": "Terms of Parole"
- },
- "title": "Mark has not satisfied a condition of his parole."
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unsave.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unsave.json
deleted file mode 100644
index a9a21f0..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unsave.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "unsave",
- "object": {
- "objectType": "article",
- "displayName": "Some article"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unshare.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unshare.json
deleted file mode 100644
index 4d45a6d..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/unshare.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "unshare",
- "object": {
- "objectType": "article",
- "displayName": "Some article"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/update.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/update.json
deleted file mode 100644
index 7dbed20..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/update.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "update",
- "object": {
- "objectType": "article",
- "displayName": "Some article"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/use.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/use.json
deleted file mode 100644
index 699c4b8..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/use.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Laura"
- },
- "verb": "use",
- "object": {
- "objectType": "product",
- "displayName": "A cool product"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/watch.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/watch.json
deleted file mode 100644
index 2052f06..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/watch.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "person",
- "displayName": "Mark"
- },
- "verb": "watch",
- "object": {
- "objectType": "video",
- "displayName": "Some random movie"
- }
-}
-
diff --git a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/win.json b/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/win.json
deleted file mode 100644
index 2cf9d5a..0000000
--- a/streams-schemas/streams-schema-activitystreams/src/test/resources/activities/win.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "$license": [
- "http://www.openwebfoundation.org/legal/the-owf-1-0-agreements/owfa-1-0",
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "actor": {
- "objectType": "organization",
- "displayName": "Arizona Diamond Backs"
- },
- "verb": "win",
- "object": {
- "objectType": "game",
- "displayName": "World Series"
- }
-}
diff --git a/streams-schemas/streams-schema-activitystreams2/pom.xml b/streams-schemas/streams-schema-activitystreams2/pom.xml
deleted file mode 100644
index a6fd8c1..0000000
--- a/streams-schemas/streams-schema-activitystreams2/pom.xml
+++ /dev/null
@@ -1,106 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schemas</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
-
- <artifactId>streams-schema-activitystreams2</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Activity Streams 2.0 schemas</description>
-
- <dependencies>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- <version>${slf4j.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>download-it-data</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
- <url>https://github.com/w3c/activitystreams/archive/master.zip</url>
- <unpack>true</unpack>
- <outputDirectory>${project.build.directory}/test-classes/w3c</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <includes>
- <include>**/*.json</include>
- <include>**/*.jsonld</include>
- <include>**/*.owl</include>
- </includes>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/streams-schemas/streams-schema-activitystreams2/src/test/java/org/w3c/activitystreams/test/ExamplesSerDeIT.java b/streams-schemas/streams-schema-activitystreams2/src/test/java/org/w3c/activitystreams/test/ExamplesSerDeIT.java
deleted file mode 100644
index 212697c..0000000
--- a/streams-schemas/streams-schema-activitystreams2/src/test/java/org/w3c/activitystreams/test/ExamplesSerDeIT.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.w3c.activitystreams.test;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.io.IOUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
-
-/**
- * Tests that activities matching core-ex* can be parsed by apache streams.
- */
-public class ExamplesSerDeIT {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ExamplesSerDeIT.class);
-
- private static final ObjectMapper MAPPER = new ObjectMapper();
-
- /**
- * Tests that activities matching core-ex* can be parsed by apache streams.
- *
- * @throws Exception test exception
- */
- @Test
- public void testCoreSerDe() throws Exception {
-
- InputStream testActivityFolderStream = ExamplesSerDeIT.class.getClassLoader()
- .getResourceAsStream("w3c/activitystreams-master/test");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- for (String file : files) {
- if ( !file.startsWith(".") && file.contains("core-ex") ) {
- LOGGER.info("File: activitystreams-master/test/" + file);
- String testFileString = new String(Files.readAllBytes(Paths.get("target/test-classes/w3c/activitystreams-master/test/" + file)));
- LOGGER.info("Content: " + testFileString);
- ObjectNode testFileObjectNode = MAPPER.readValue(testFileString, ObjectNode.class);
- LOGGER.info("Object:" + testFileObjectNode);
- }
- }
- }
-
- /**
- * Tests that activities matching simple* can be parsed by apache streams.
- *
- * @throws Exception test exception
- */
- @Test
- public void testSimpleSerDe() throws Exception {
-
- InputStream testActivityFolderStream = ExamplesSerDeIT.class.getClassLoader()
- .getResourceAsStream("w3c/activitystreams-master/test");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- for (String file : files) {
- if ( !file.startsWith(".") && file.contains("simple") ) {
- LOGGER.info("File: activitystreams-master/test/" + file);
- String testFileString = new String(Files.readAllBytes(Paths.get("target/test-classes/w3c/activitystreams-master/test/" + file)));
- LOGGER.info("Content: " + testFileString);
- ObjectNode testFileObjectNode = MAPPER.readValue(testFileString, ObjectNode.class);
- LOGGER.info("Object:" + testFileObjectNode);
- }
- }
- }
-
- /**
- * Tests that activities matching vocabulary-ex* can be parsed by apache streams.
- *
- * @throws Exception test exception
- */
- @Test( enabled = false )
- public void testVocabularySerDe() throws Exception {
-
- InputStream testActivityFolderStream = ExamplesSerDeIT.class.getClassLoader()
- .getResourceAsStream("w3c/activitystreams-master/test");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- for (String file : files) {
- if ( !file.startsWith(".") && file.contains("vocabulary-ex") ) {
- LOGGER.info("File: activitystreams-master/test/" + file);
- String testFileString = new String(Files.readAllBytes(Paths.get("target/test-classes/w3c/activitystreams-master/test/" + file)));
- LOGGER.info("Content: " + testFileString);
- ObjectNode testFileObjectNode = MAPPER.readValue(testFileString, ObjectNode.class);
- LOGGER.info("Object:" + testFileObjectNode);
- }
- }
- }
-
- /**
- * Tests that activities expect to fail cannot be parsed by apache streams.
- *
- * @throws Exception test exception
- */
- @Test( enabled = false )
- public void testFailSerDe() throws Exception {
-
- InputStream testActivityFolderStream = ExamplesSerDeIT.class.getClassLoader()
- .getResourceAsStream("w3c/activitystreams-master/test/fail");
- List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);
-
- for (String file : files) {
- if ( !file.startsWith(".") && file.contains("vocabulary-ex") ) {
- LOGGER.info("File: activitystreams-master/test/fail/" + file);
- String testFileString = new String(Files.readAllBytes(Paths.get("target/test-classes/w3c/activitystreams-master/test/" + file)));
- LOGGER.info("Content: " + testFileString);
- ObjectNode testFileObjectNode = MAPPER.readValue(testFileString, ObjectNode.class);
- LOGGER.info("Object:" + testFileObjectNode);
- }
- }
- }
-}
\ No newline at end of file
diff --git a/streams-testing/README.md b/streams-testing/README.md
deleted file mode 100644
index 0322c32..0000000
--- a/streams-testing/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-testing
-==================================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-testing/pom.xml b/streams-testing/pom.xml
deleted file mode 100644
index bf6bad2..0000000
--- a/streams-testing/pom.xml
+++ /dev/null
@@ -1,57 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>apache-streams</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-testing</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Apache Streams Testing Configuration</description>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>com.carrotsearch.randomizedtesting</groupId>
- <artifactId>randomizedtesting-runner</artifactId>
- <version>2.5.3</version>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-testing/src/test/resources/log4j.properties b/streams-testing/src/test/resources/log4j.properties
deleted file mode 100644
index c183d78..0000000
--- a/streams-testing/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-log4j.logger.akka=WARN
-log4j.logger.ch.qos.logback=ERROR
-log4j.logger.com.foursquare=WARN
-log4j.logger.com.google=WARN
-log4j.logger.com.mongodb=WARN
-log4j.logger.org.apache.streams.youtube=WARN
-log4j.logger.io.dropwizard=WARN
-log4j.logger.org.apache.streams=WARN
-log4j.logger.org.apache.spark=WARN
-log4j.logger.org.apache.hadoop=ERROR
-log4j.logger.org.apache.pig=WARN
-log4j.logger.org.apache.http=WARN
-log4j.logger.org.eclipse.jetty=WARN
-log4j.logger.org.elasticsearch=WARN
-log4j.logger.org.reflections=WARN
-log4j.logger.org.spark-project.jetty=WARN
-log4j.logger.Remoting=WARN
diff --git a/streams-testing/src/test/resources/logback-test.xml b/streams-testing/src/test/resources/logback-test.xml
deleted file mode 100644
index 9dd8bf3..0000000
--- a/streams-testing/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,45 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
--->
-<configuration>
- <logger name="akka" level="WARN"/>
- <logger name="ch.qos.logback" level="ERROR"/>
- <logger name="com.mongodb" level="WARN"/>
- <logger name="com.foursquare" level="WARN"/>
- <logger name="com.google" level="WARN"/>
- <logger name="org.apache.streams.youtube" level="WARN"/>
- <logger name="io.dropwizard" level="WARN"/>
- <logger name="org.apache.streams" level="WARN"/>
- <logger name="org.apache.spark" level="WARN"/>
- <logger name="org.apache.hadoop" level="ERROR"/>
- <logger name="org.apache.pig" level="WARN"/>
- <logger name="org.apache.http" level="WARN"/>
- <logger name="org.elasticsearch" level="WARN"/>
- <logger name="org.reflections" level="WARN"/>
- <logger name="org.spark-project.jetty" level="WARN"/>
- <logger name="org.eclipse.jetty" level="WARN"/>
- <logger name="Remoting" level="WARN"/>
- <appender name="STDOUT"
- class="ch.qos.logback.core.ConsoleAppender"> <!-- encoders are assigned the type ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
- <encoder>
- <pattern>%-4relative [%thread] %-5level %logger{35} - %msg %n</pattern>
- </encoder>
- </appender>
- <root level="INFO">
- <appender-ref ref="STDOUT"/>
- </root>
-</configuration>
diff --git a/streams-util/README.md b/streams-util/README.md
deleted file mode 100644
index 6d93b8b..0000000
--- a/streams-util/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-util
-===============================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-util/pom.xml b/streams-util/pom.xml
deleted file mode 100644
index 9ac463b..0000000
--- a/streams-util/pom.xml
+++ /dev/null
@@ -1,179 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>apache-streams</artifactId>
- <groupId>org.apache.streams</groupId>
- <version>0.5.1-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>streams-util</artifactId>
- <name>${project.artifactId}</name>
-
- <description>Apache Streams Utilities</description>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.github.wnameless</groupId>
- <artifactId>json-flattener</artifactId>
- </dependency>
- <dependency>
- <groupId>com.carrotsearch.randomizedtesting</groupId>
- <artifactId>randomizedtesting-runner</artifactId>
- <version>2.5.3</version>
- </dependency>
- <dependency>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-core</artifactId>
- <version>${jsonschema2pojo.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-schema-activitystreams</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-resource</goal>
- </goals>
- <configuration>
- <resources>
- <resource>
- <directory>${project.basedir}/src/main/jsonschema</directory>
- </resource>
- <resource>
- <directory>src/main/xmlschema</directory>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>unpack-schemas</id>
- <phase>process-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-schemas</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>jar</includeTypes>
- </configuration>
- </execution>
- <execution>
- <id>unpack-testdocs</id>
- <phase>process-resources</phase>
- <goals>
- <goal>unpack-dependencies</goal>
- </goals>
- <configuration>
- <includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/activitystreams-testdocs</outputDirectory>
- <includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
- <includeTypes>test-jar</includeTypes>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
diff --git a/streams-util/src/main/java/org/apache/streams/util/ComponentUtils.java b/streams-util/src/main/java/org/apache/streams/util/ComponentUtils.java
deleted file mode 100644
index 26a6284..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/ComponentUtils.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.management.ManagementFactory;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import javax.management.InstanceAlreadyExistsException;
-import javax.management.MBeanRegistrationException;
-import javax.management.MBeanServer;
-import javax.management.MalformedObjectNameException;
-import javax.management.NotCompliantMBeanException;
-import javax.management.ObjectName;
-
-/**
- * Common utilities for Streams components.
- */
-public class ComponentUtils {
- private static final Logger LOGGER = LoggerFactory.getLogger(ComponentUtils.class);
-
- /**
- * Certain types of queues will fail to {@link java.util.Queue#offer(Object)} an item due to many factors
- * depending on the type of queue. <code>offerUntilSuccess</code> will not return until the item has been
- * successfully queued onto the desired queue
- * @param entry item to queue
- * @param queue queue to add the entry to
- * @param <T> type
- */
- public static <T> void offerUntilSuccess(T entry, Queue<T> queue) {
- boolean success;
- do {
- success = queue.offer(entry);
- Thread.yield();
- }
- while ( !success );
- }
-
- /**
- * Certain types of queues will return null when calling {@link java.util.Queue#poll()} due to many factors depending
- * on the type of queue. <code>pollWhileNotEmpty</code> will poll the queue until an item from the queue is returned
- * or the queue is empty. If the queue is empty it will return NULL.
- * @param queue queue to read the entry from
- * @param <T> type
- * @return result
- */
- public static <T> T pollWhileNotEmpty(Queue<T> queue) {
- T item = queue.poll();
- while (!queue.isEmpty() && item == null) {
- Thread.yield();
- item = queue.poll();
- }
- return item;
- }
-
- /**
- * Attempts to safely {@link java.util.concurrent.ExecutorService#shutdown()}
- * and {@link java.util.concurrent.ExecutorService#awaitTermination(long, java.util.concurrent.TimeUnit)}
- * of an {@link java.util.concurrent.ExecutorService}.
- * @param stream service to be shutdown
- * @param initialWait time in seconds to wait for currently running threads to finish execution
- * @param secondaryWait time in seconds to wait for running threads that did not terminate to acknowledge their forced termination
- */
- public static void shutdownExecutor(ExecutorService stream, int initialWait, int secondaryWait) {
- stream.shutdown();
- try {
- if (!stream.awaitTermination(initialWait, TimeUnit.SECONDS)) {
- stream.shutdownNow();
- if (!stream.awaitTermination(secondaryWait, TimeUnit.SECONDS)) {
- LOGGER.error("Executor Service did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- stream.shutdownNow();
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * Removes all mbeans registered undered a specific domain. Made specificly to clean up at unit tests
- * @param domain mbean domain
- */
- public static void removeAllMBeansOfDomain(String domain) throws Exception {
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- domain = domain.endsWith(":") ? domain : domain + ":";
- ObjectName objectName = new ObjectName(domain + "*");
- Set<ObjectName> mbeanNames = mbs.queryNames(objectName, null);
- for (ObjectName name : mbeanNames) {
- mbs.unregisterMBean(name);
- }
- }
-
- /**
- * Attempts to register an object with local MBeanServer. Throws runtime exception on errors.
- * @param name name to register bean with
- * @param mbean mbean to register
- */
- public static <V> void registerLocalMBean(String name, V mbean) {
- try {
- ObjectName objectName = new ObjectName(name);
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- mbs.registerMBean(mbean, objectName);
- } catch (MalformedObjectNameException | InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException ex) {
- LOGGER.error("Failed to register MXBean : {}", ex);
- throw new RuntimeException(ex);
- }
- }
-
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/GuidUtils.java b/streams-util/src/main/java/org/apache/streams/util/GuidUtils.java
deleted file mode 100644
index 3bc77df..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/GuidUtils.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util;
-
-import com.google.common.hash.Hashing;
-import org.apache.commons.lang3.StringUtils;
-
-import java.nio.charset.Charset;
-import java.util.Arrays;
-import java.util.Objects;
-
-/**
- * GuidUtils contains methods for generating guids from identifiers.
- */
-public class GuidUtils {
-
- private static final Charset UTF8_CHARSET = Charset.forName("UTF-8");
-
- /**
- * generateGuid from list of parts.
- * @param parts list of parts
- * @return guid
- */
- public static String generateGuid(String... parts) {
-
- StringBuilder seed = new StringBuilder();
-
- for ( String part : parts ) {
- Objects.requireNonNull(part);
- if(StringUtils.isNotBlank(part)) {
- seed.append(part);
- }
- }
-
- return Arrays.toString(Hashing.goodFastHash(24).hashString(seed, UTF8_CHARSET).asBytes());
-
- }
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/PropertyUtil.java b/streams-util/src/main/java/org/apache/streams/util/PropertyUtil.java
deleted file mode 100644
index a394b92..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/PropertyUtil.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.fasterxml.jackson.databind.node.ValueNode;
-import com.github.wnameless.json.flattener.JsonFlattener;
-import com.github.wnameless.json.unflattener.JsonUnflattener;
-import org.apache.commons.lang3.StringUtils;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Class transforms nested properties of activities, actors, objects, etc...
- */
-public class PropertyUtil {
-
- private static final PropertyUtil INSTANCE = new PropertyUtil(new ObjectMapper());
-
- public static PropertyUtil getInstance() {
- return INSTANCE;
- }
-
- public static PropertyUtil getInstance(ObjectMapper mapper) {
- return new PropertyUtil(mapper);
- }
-
- private ObjectMapper mapper;
-
- public PropertyUtil(ObjectMapper mapper) {
- this.mapper = mapper;
- }
-
- public Map<String, Object> flattenToMap(ObjectNode object) {
- return flattenToMap(mapper, object);
- }
-
- public ObjectNode flattenToObjectNode(ObjectNode object) {
- return flattenToObjectNode(mapper, object);
- }
-
- public ObjectNode unflattenMap(Map<String, Object> object) {
- return unflattenMap(mapper, object);
- }
-
- public ObjectNode unflattenObjectNode(ObjectNode object) {
- return unflattenObjectNode(mapper, object);
- }
-
- public static Map<String, Object> flattenToMap(ObjectMapper mapper, ObjectNode object) {
- Map<String, Object> flatObject;
- try {
- flatObject = mapper.readValue(JsonFlattener.flatten(mapper.writeValueAsString(object)), Map.class);
- } catch( Exception ex ) {
- return null;
- }
- return flatObject;
- }
-
- public static ObjectNode flattenToObjectNode(ObjectMapper mapper, ObjectNode object) {
- ObjectNode flatObjectNode;
- try {
- flatObjectNode = mapper.readValue(JsonFlattener.flatten(mapper.writeValueAsString(object)), ObjectNode.class);
- } catch( Exception ex ) {
- return null;
- }
- return flatObjectNode;
- }
-
- public static ObjectNode unflattenMap(ObjectMapper mapper, Map<String, Object> object) {
- ObjectNode unflatObjectNode;
- try {
- unflatObjectNode = mapper.readValue(JsonUnflattener.unflatten(mapper.writeValueAsString(object)), ObjectNode.class);
- } catch( Exception ex ) {
- return null;
- }
- return unflatObjectNode;
- }
-
- public static ObjectNode unflattenObjectNode(ObjectMapper mapper, ObjectNode object) {
- ObjectNode unflatObjectNode;
- try {
- unflatObjectNode = mapper.readValue(JsonUnflattener.unflatten(mapper.writeValueAsString(object)), ObjectNode.class);
- } catch( Exception ex ) {
- return null;
- }
- return unflatObjectNode;
- }
-
-
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/SerializationUtil.java b/streams-util/src/main/java/org/apache/streams/util/SerializationUtil.java
deleted file mode 100644
index 4a4d831..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/SerializationUtil.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util;
-
-import org.apache.commons.io.input.ClassLoaderObjectInputStream;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
-/**
- * SerializationUtil contains methods for serializing, deserializing, and cloning
- * documents and tasks.
- */
-public class SerializationUtil {
-
- /**
- * serialize Object as byte array.
- *
- * <p/>
- * BORROwED FROM APACHE STORM PROJECT
- *
- * @param obj Object
- * @return byte[]
- */
- public static byte[] serialize(Object obj) {
- try {
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- ObjectOutputStream oos = new ObjectOutputStream(bos);
- oos.writeObject(obj);
- oos.close();
- return bos.toByteArray();
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
- }
- }
-
- /**
- * deserialize byte array as Object.
- *
- * <p/>
- * BORROwED FROM APACHE STORM PROJECT
- *
- * @param serialized byte[]
- * @return Object
- */
- public static Object deserialize(byte[] serialized) {
- try {
- ByteArrayInputStream bis = new ByteArrayInputStream(serialized);
- ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
- ObjectInputStream ois = new ClassLoaderObjectInputStream(classLoader, bis);
- Object ret = ois.readObject();
- ois.close();
- return ret;
- } catch (IOException | ClassNotFoundException ioe) {
- throw new RuntimeException(ioe);
- }
- }
-
- /**
- * clone Object by serialization.
- * @param obj Object
- * @param <T> type
- * @return cloned Object
- */
- public static <T> T cloneBySerialization(T obj) {
- if ( obj != null ) {
- return (T) deserialize(serialize(obj));
- } else {
- return null;
- }
- }
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/AbstractBackOffStrategy.java b/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/AbstractBackOffStrategy.java
deleted file mode 100644
index 3dc3e08..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/AbstractBackOffStrategy.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.util.api.requests.backoff;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * @see org.apache.streams.util.api.requests.backoff.BackOffStrategy
- */
-public abstract class AbstractBackOffStrategy implements BackOffStrategy {
-
- private long baseSleepTime;
- private long lastSleepTime;
- private int maxAttempts;
- private AtomicInteger attemptsCount;
-
- /**
- * A BackOffStrategy that can effectively be used endlessly.
- * @param baseBackOffTime amount of time back of in seconds
- */
- public AbstractBackOffStrategy(long baseBackOffTime) {
- this(baseBackOffTime, -1);
- }
-
- /**
- * A BackOffStrategy that has a limited number of uses before it throws a
- * {@link org.apache.streams.util.api.requests.backoff.BackOffException}.
- * @param baseBackOffTime time to back off in milliseconds, must be greater than 0.
- * @param maximumNumberOfBackOffAttempts maximum number of attempts, must be grater than 0 or -1.
- * -1 indicates there is no maximum number of attempts.
- */
- public AbstractBackOffStrategy(long baseBackOffTime, int maximumNumberOfBackOffAttempts) {
- if (baseBackOffTime <= 0) {
- throw new IllegalArgumentException("backOffTimeInMilliSeconds is not greater than 0 : " + baseBackOffTime);
- }
- if (maximumNumberOfBackOffAttempts <= 0 && maximumNumberOfBackOffAttempts != -1) {
- throw new IllegalArgumentException("maximumNumberOfBackOffAttempts is not greater than 0 : " + maximumNumberOfBackOffAttempts);
- }
- this.baseSleepTime = baseBackOffTime;
- this.maxAttempts = maximumNumberOfBackOffAttempts;
- this.attemptsCount = new AtomicInteger(0);
- }
-
- @Override
- public void backOff() throws BackOffException {
- int attempt = this.attemptsCount.getAndIncrement();
- if (attempt >= this.maxAttempts && this.maxAttempts != -1) {
- throw new BackOffException(attempt, this.lastSleepTime);
- } else {
- try {
- Thread.sleep(this.lastSleepTime = calculateBackOffTime(attempt, this.baseSleepTime));
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- }
- }
-
- @Override
- public void reset() {
- this.attemptsCount.set(0);
- }
-
- /**
- * Calculate the amount of time in milliseconds that the strategy should back off for
- * @param attemptCount the number of attempts the strategy has backed off.
- * i.e. 1 -> this is the first attempt, 2 -> this is the second attempt, etc.
- * @param baseSleepTime the minimum amount of time it should back off for in milliseconds
- * @return the amount of time it should back off in milliseconds
- */
- protected abstract long calculateBackOffTime(int attemptCount, long baseSleepTime);
-
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/BackOffException.java b/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/BackOffException.java
deleted file mode 100644
index 692c0b6..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/BackOffException.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.util.api.requests.backoff;
-
-/**
- * Exception that is thrown when a {@link AbstractBackOffStrategy} has attempted to
- * <code>backOff()</code> more than the {@link AbstractBackOffStrategy} was configured for.
- */
-public class BackOffException extends Exception {
-
- private int attemptCount;
- private long sleepTime;
-
- public BackOffException() {
- this(-1, -1);
- }
-
- public BackOffException(String message) {
- this(message, -1, -1);
- }
-
- public BackOffException(int attemptCount, long maxSleepTime) {
- this.attemptCount = attemptCount;
- this.sleepTime = maxSleepTime;
- }
-
- /**
- * BackOffException constructor.
- * @param message message
- * @param attemptCount attemptCount
- * @param maxSleepTime maxSleepTime (in millis)
- */
- public BackOffException(String message, int attemptCount, long maxSleepTime) {
- super(message);
- this.attemptCount = attemptCount;
- this.sleepTime = maxSleepTime;
- }
-
- /**
- * Gets the number of back off attempts that happened before the exception was thrown. If the function that
- * initialized this exception does not set the number of attempts, -1 will be returned.
- * @return number of back off attempts
- */
- public int getNumberOfBackOffsAttempted() {
- return this.attemptCount;
- }
-
- /**
- * Gets the longest sleep period that the strategy attempted. If the function that
- * initialized this exception does not set the longest sleep period, -1 will be returned.
- * @return longest sleep period that the strategy attempted
- */
- public long getLongestBackOff() {
- return this.sleepTime;
- }
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/BackOffStrategy.java b/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/BackOffStrategy.java
deleted file mode 100644
index 44497ab..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/BackOffStrategy.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.util.api.requests.backoff;
-
-/**
- * BackOffStrategy will cause the current thread to sleep for a specific amount of time. This is used to adhere to
- * api rate limits.
- *
- * <p/>
- * The example below illustrates using a BackOffStrategy to slow down requests when you hit a rate limit exception.
- *
- * <code>
- * public void pollApi(ApiClient apiClient, BackOffStrategy backOffStrategy) throws BackOffException {
- * while( apiClient.hasMoreData() ) {
- * try {
- * apiClient.pollData();
- * } catch (RateLimitException rle) {
- * backOffStrategy.backOff();
- * }
- * }
- * }
- * </code>
- *
- */
-public interface BackOffStrategy {
-
- /**
- * Cause the current thread to sleep for an amount of time based on the implemented strategy. If limits are set
- * on the number of times the backOff can be called, an exception will be thrown.
- * @throws BackOffException BackOffException
- */
- public void backOff() throws BackOffException;
-
- /**
- * Rests the back off strategy to its original state.
- * After the call the strategy will act as if {@link AbstractBackOffStrategy#backOff()}
- * has never been called.
- */
- public void reset();
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/impl/ConstantTimeBackOffStrategy.java b/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/impl/ConstantTimeBackOffStrategy.java
deleted file mode 100644
index 26ec225..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/impl/ConstantTimeBackOffStrategy.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.util.api.requests.backoff.impl;
-
-import org.apache.streams.util.api.requests.backoff.AbstractBackOffStrategy;
-
-/**
- * A {@link org.apache.streams.util.api.requests.backoff.AbstractBackOffStrategy} that causes the current thread to sleep the
- * same amount of time each time <code>backOff()</code> is called.
- *
- */
-public class ConstantTimeBackOffStrategy extends AbstractBackOffStrategy {
-
- /**
- * A ConstantTimeBackOffStrategy that can effectively be used endlessly.
- * @param baseBackOffTimeInMiliseconds amount of time back of in milliseconds
- */
- public ConstantTimeBackOffStrategy(long baseBackOffTimeInMiliseconds) {
- this(baseBackOffTimeInMiliseconds, -1);
- }
-
- /**
- * A ConstantTimeBackOffStrategy that has a limited number of uses before it
- * throws a {@link org.apache.streams.util.api.requests.backoff.BackOffException}
- * @param baseBackOffTimeInMiliseconds time to back off in milliseconds, must be greater than 0.
- * @param maximumNumberOfBackOffAttempts maximum number of attempts, must be grater than 0 or -1.
- * -1 indicates there is no maximum number of attempts.
- */
- public ConstantTimeBackOffStrategy(long baseBackOffTimeInMiliseconds, int maximumNumberOfBackOffAttempts) {
- super(baseBackOffTimeInMiliseconds, maximumNumberOfBackOffAttempts);
- }
-
- @Override
- protected long calculateBackOffTime(int attemptCount, long baseSleepTime) {
- return baseSleepTime;
- }
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/impl/ExponentialBackOffStrategy.java b/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/impl/ExponentialBackOffStrategy.java
deleted file mode 100644
index 0962984..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/impl/ExponentialBackOffStrategy.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.util.api.requests.backoff.impl;
-
-import org.apache.streams.util.api.requests.backoff.AbstractBackOffStrategy;
-
-/**
- * Exponential backoff strategy. Calculated by baseBackOffTimeInSeconds raised the attempt-count power.
- */
-public class ExponentialBackOffStrategy extends AbstractBackOffStrategy {
-
- /**
- * Unlimited use ExponentialBackOffStrategy.
- * @param baseBackOffTimeInSeconds baseBackOffTimeInSeconds
- */
- public ExponentialBackOffStrategy(int baseBackOffTimeInSeconds) {
- this(baseBackOffTimeInSeconds, -1);
- }
-
- /**
- * Limited use ExponentialBackOffStrategy.
- * @param baseBackOffTimeInSeconds baseBackOffTimeInSeconds
- * @param maxNumAttempts maxNumAttempts
- */
- public ExponentialBackOffStrategy(int baseBackOffTimeInSeconds, int maxNumAttempts) {
- super(baseBackOffTimeInSeconds, maxNumAttempts);
- }
-
- @Override
- protected long calculateBackOffTime(int attemptCount, long baseSleepTime) {
- return Math.round(Math.pow(baseSleepTime, attemptCount)) * 1000;
- }
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/impl/LinearTimeBackOffStrategy.java b/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/impl/LinearTimeBackOffStrategy.java
deleted file mode 100644
index d6f323f..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/api/requests/backoff/impl/LinearTimeBackOffStrategy.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.util.api.requests.backoff.impl;
-
-import org.apache.streams.util.api.requests.backoff.AbstractBackOffStrategy;
-
-/**
- * A {@link org.apache.streams.util.api.requests.backoff.AbstractBackOffStrategy} that causes back offs in linear increments. Each
- * attempt cause an increase back off period.
- * Calculated by attemptNumber * baseBackOffAmount.
- */
-public class LinearTimeBackOffStrategy extends AbstractBackOffStrategy {
-
- public LinearTimeBackOffStrategy(int baseBackOffTimeInSeconds) {
- this(baseBackOffTimeInSeconds, -1);
- }
-
- public LinearTimeBackOffStrategy(int baseBackOffTimeInSeconds, int maxAttempts) {
- super(baseBackOffTimeInSeconds, -1);
- }
-
- @Override
- protected long calculateBackOffTime(int attemptCount, long baseSleepTime) {
- return 1000L * attemptCount * baseSleepTime;
- }
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/oauth/tokens/AbstractOauthToken.java b/streams-util/src/main/java/org/apache/streams/util/oauth/tokens/AbstractOauthToken.java
deleted file mode 100644
index 41ec4b6..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/oauth/tokens/AbstractOauthToken.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.util.oauth.tokens;
-
-/**
- * AbstractOauthToken.
- */
-public abstract class AbstractOauthToken {
-
- /**
- * Must create equals method for all OauthTokens.
- * @param object object for comparison
- * @return true if equal, and false otherwise
- */
- protected abstract boolean internalEquals(Object object);
-
- @Override
- public boolean equals(Object object) {
- return this.internalEquals(object);
- }
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/oauth/tokens/tokenmanager/SimpleTokenManager.java b/streams-util/src/main/java/org/apache/streams/util/oauth/tokens/tokenmanager/SimpleTokenManager.java
deleted file mode 100644
index 7b3f370..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/oauth/tokens/tokenmanager/SimpleTokenManager.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.util.oauth.tokens.tokenmanager;
-
-import java.util.Collection;
-
-/**
- * Manages access to oauth tokens. Allows a caller to add tokens to the token pool and receive an available token.
- */
-public interface SimpleTokenManager<T> {
-
- /**
- * Adds a token to the available token pool.
- * @param token Token to be added
- * @return true, if token was successfully added to the pool and false otherwise.
- */
- public boolean addTokenToPool(T token);
-
- /**
- * Adds a {@link java.util.Collection} of tokens to the available token pool.
- * @param tokens Tokens to be added
- * @return true, if the token pool size increased after adding the tokens, and false otherwise.
- */
- public boolean addAllTokensToPool(Collection<T> tokens);
-
- /**
- * Get an available token. If no tokens are available it returns null.
- * @return next available token
- */
- public T getNextAvailableToken();
-
- /**
- * Get the number of available tokens.
- * @return number of available tokens
- */
- public int numAvailableTokens();
-
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/oauth/tokens/tokenmanager/impl/BasicTokenManager.java b/streams-util/src/main/java/org/apache/streams/util/oauth/tokens/tokenmanager/impl/BasicTokenManager.java
deleted file mode 100644
index d69c61f..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/oauth/tokens/tokenmanager/impl/BasicTokenManager.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance *
-http://www.apache.org/licenses/LICENSE-2.0 *
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License. */
-
-package org.apache.streams.util.oauth.tokens.tokenmanager.impl;
-
-import org.apache.streams.util.oauth.tokens.tokenmanager.SimpleTokenManager;
-
-import java.util.ArrayList;
-import java.util.Collection;
-
-/**
- * Manages a pool of tokens the most basic possible way.
- * If all tokens are added to the manager before {@link BasicTokenManager#getNextAvailableToken() getNextAvailableToken}
- * is called tokens are issued in the order they were added to the manager, FIFO. The BasicTokenManager acts as a circular queue
- * of tokens. Once the manager issues all available tokens it will cycle back to the first token and start issuing tokens again.
- *
- * </p>
- * When adding tokens to the pool of available tokens, the manager will not add tokens that are already in the pool.
- *
- * <p/>
- * The manager class is thread safe.
- */
-public class BasicTokenManager<T> implements SimpleTokenManager<T> {
-
- private ArrayList<T> availableTokens;
- private int nextToken;
-
- public BasicTokenManager() {
- this(null);
- }
-
- /**
- * BasicTokenManager constructor.
- * @param tokens Collection of tokens
- */
- public BasicTokenManager(Collection<T> tokens) {
- if (tokens != null) {
- this.availableTokens = new ArrayList<>(tokens.size());
- this.addAllTokensToPool(tokens);
- } else {
- this.availableTokens = new ArrayList<>();
- }
- this.nextToken = 0;
- }
-
- @Override
- public synchronized boolean addTokenToPool(T token) {
- return !(token == null || this.availableTokens.contains(token)) && this.availableTokens.add(token);
- }
-
- @Override
- public synchronized boolean addAllTokensToPool(Collection<T> tokens) {
- int startSize = this.availableTokens.size();
- for (T token : tokens) {
- this.addTokenToPool(token);
- }
- return startSize < this.availableTokens.size();
- }
-
- @Override
- public synchronized T getNextAvailableToken() {
- T token;
- if (this.availableTokens.size() == 0) {
- return null;
- } else {
- token = this.availableTokens.get(nextToken++);
- if (nextToken == this.availableTokens.size()) {
- nextToken = 0;
- }
- return token;
- }
- }
-
- @Override
- public synchronized int numAvailableTokens() {
- return this.availableTokens.size();
- }
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/schema/FieldType.java b/streams-util/src/main/java/org/apache/streams/util/schema/FieldType.java
deleted file mode 100644
index 57a1d44..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/schema/FieldType.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.schema;
-
-/**
- * FieldType defines xsd types that streams schema processing libraries should
- * be able to translate.
- */
-public enum FieldType {
- STRING,
- INTEGER,
- NUMBER,
- BOOLEAN,
- OBJECT,
- ARRAY
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/schema/FieldUtil.java b/streams-util/src/main/java/org/apache/streams/util/schema/FieldUtil.java
deleted file mode 100644
index acbe12f..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/schema/FieldUtil.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.schema;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-/**
- * FieldUtil contains methods to assist in understanding fields defined within schemas.
- */
-public class FieldUtil {
-
- /**
- * determine FieldType from ObjectNode.
- * @param fieldNode ObjectNode
- * @return FieldType
- */
- public static FieldType determineFieldType(ObjectNode fieldNode) {
- String typeSchemaField = "type";
- if ( !fieldNode.has(typeSchemaField)) {
- return null;
- }
- String typeSchemaFieldValue = fieldNode.get(typeSchemaField).asText();
- switch (typeSchemaFieldValue) {
- case "string":
- return FieldType.STRING;
- case "integer":
- return FieldType.INTEGER;
- case "number":
- return FieldType.NUMBER;
- case "object":
- return FieldType.OBJECT;
- case "boolean":
- return FieldType.BOOLEAN;
- case "array":
- return FieldType.ARRAY;
- default:
- return null;
- }
- }
-
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/schema/FileUtil.java b/streams-util/src/main/java/org/apache/streams/util/schema/FileUtil.java
deleted file mode 100644
index 9de044a..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/schema/FileUtil.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.schema;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * FileUtil contains methods to assist in working with local schema files during
- * source or resource generation.
- */
-public class FileUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FileUtil.class);
-
- /**
- * drop source path prefix between inputFile and sourceDirectory.
- * @param inputFile inputFile
- * @param sourceDirectory sourceDirectory
- * @return without path prefix
- */
- public static String dropSourcePathPrefix(String inputFile, String sourceDirectory) {
- if (StringUtils.isBlank(sourceDirectory)) {
- return inputFile;
- } else {
- try {
- if ( inputFile.contains(sourceDirectory) && inputFile.indexOf(sourceDirectory) > 0) {
- return inputFile.substring(inputFile.indexOf(sourceDirectory) + sourceDirectory.length() + 1);
- }
- } catch ( Throwable throwable ) {
- return inputFile;
- }
- }
- return inputFile;
- }
-
- /**
- * swapExtension.
- * @param inputFile inputFile
- * @param originalExtension originalExtension
- * @param newExtension newExtension
- * @return extension swapped
- */
- public static String swapExtension(String inputFile, String originalExtension, String newExtension) {
- if (inputFile.endsWith("." + originalExtension)) {
- return inputFile.replace("." + originalExtension, "." + newExtension);
- } else {
- return inputFile;
- }
- }
-
- /**
- * dropExtension.
- * @param inputFile inputFile
- * @return extension dropped
- */
- public static String dropExtension(String inputFile) {
- if (inputFile.contains(".")) {
- return inputFile.substring(0, inputFile.lastIndexOf("."));
- } else {
- return inputFile;
- }
- }
-
- /**
- * writeFile.
- * @param resourceFile resourceFile
- * @param resourceContent resourceContent
- */
- public static void writeFile(String resourceFile, String resourceContent) {
- try {
- File path = new File(resourceFile);
- File dir = path.getParentFile();
- if ( !dir.exists() ) {
- dir.mkdirs();
- }
- Files.write(Paths.get(resourceFile), resourceContent.getBytes(), StandardOpenOption.CREATE_NEW);
- } catch (Exception ex) {
- LOGGER.error("Write Exception: {}", ex);
- }
- }
-
- /**
- * resolveRecursive.
- * @param config GenerationConfig
- * @param schemaFiles List of schemaFiles
- */
- public static void resolveRecursive(GenerationConfig config, List<File> schemaFiles) {
-
- Preconditions.checkArgument(schemaFiles.size() > 0);
- int index = 0;
- while ( schemaFiles.size() > index) {
- File child = schemaFiles.get(index);
- if (child.isDirectory()) {
- schemaFiles.addAll(Arrays.asList(child.listFiles(config.getFileFilter())));
- schemaFiles.remove(child);
- } else {
- index += 1;
- }
- }
-
- }
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/schema/GenerationConfig.java b/streams-util/src/main/java/org/apache/streams/util/schema/GenerationConfig.java
deleted file mode 100644
index d7fa7e7..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/schema/GenerationConfig.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.schema;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.net.URL;
-import java.util.Iterator;
-
-/**
- * GenerationConfig represents the common fields and field accessors for
- * streams modules that transform schemas into generated-sources or generated-resources.
- */
-public interface GenerationConfig {
-
- /**
- * Gets the 'source' configuration option.
- *
- * @return The source file(s) or directory(ies) from which JSON Schema will
- * be read.
- */
- Iterator<URL> getSource();
-
- /**
- * Gets the 'targetDirectory' configuration option.
- *
- * @return The target directory into which generated types will be written
- * (may or may not exist before types are written)
- */
- File getTargetDirectory();
-
- /**
- * Gets the 'outputEncoding' configuration option.
- *
- * @return The character encoding that should be used when writing output files.
- */
- String getOutputEncoding();
-
- /**
- * Gets the file filter used to isolate the schema mapping files in the
- * source directories.
- *
- * @return the file filter use when scanning for schema files.
- */
- FileFilter getFileFilter();
-
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/schema/Schema.java b/streams-util/src/main/java/org/apache/streams/util/schema/Schema.java
deleted file mode 100644
index a9517c1..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/schema/Schema.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.schema;
-
-import com.fasterxml.jackson.databind.JsonNode;
-
-import java.net.URI;
-
-/**
- * A JSON Schema document.
- */
-public class Schema {
-
- private final URI id;
- private final URI uri;
- private final JsonNode content;
- private final Schema parent;
- private final boolean generate;
-
- /**
- * Schema constructor.
- * @param uri uri
- * @param content JsonNode content
- * @param parent Schema parent
- * @param generate whether to generate
- */
- public Schema(URI uri, JsonNode content, Schema parent, boolean generate) {
- this.uri = uri;
- this.content = content;
- this.parent = parent;
- this.generate = generate;
- this.id = content.has("id") ? URI.create(content.get("id").asText()) : null;
- }
-
- public URI getId() {
- return id;
- }
-
- public URI getUri() {
- return uri;
- }
-
- public JsonNode getContent() {
- return content;
- }
-
- /**
- * getParentContent.
- * @return Parent.Content
- */
- public JsonNode getParentContent() {
- if ( parent != null ) {
- return parent.getContent();
- } else {
- return null;
- }
- }
-
- /**
- * getParentUri.
- * @return Parent.Uri
- */
- public URI getParentUri() {
- if ( parent != null ) {
- return parent.getUri();
- } else {
- return null;
- }
- }
-
- public boolean isGenerated() {
- return generate;
- }
-
- public Schema getParent() {
- return parent;
- }
-
-}
\ No newline at end of file
diff --git a/streams-util/src/main/java/org/apache/streams/util/schema/SchemaStore.java b/streams-util/src/main/java/org/apache/streams/util/schema/SchemaStore.java
deleted file mode 100644
index 5bdf8b0..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/schema/SchemaStore.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.schema;
-
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-import java.net.URI;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.Optional;
-
-/**
- * A SchemaStore resolves and indexes json schemas and makes their properties available.
- *
- * <p/>
- * Implementations include
- * - SchemaStoreImpl
- */
-public interface SchemaStore extends Comparator<Schema> {
-
- Schema create(URI uri);
-
- Schema create(Schema parent, String path);
-
- void clearCache();
-
- Integer getSize();
-
- Optional<Schema> getById(URI id);
-
- Optional<Schema> getByUri(URI uri);
-
- Integer getFileUriCount();
-
- Integer getHttpUriCount();
-
- Iterator<Schema> getSchemaIterator();
-
- ObjectNode resolveProperties(Schema schema, ObjectNode fieldNode, String resourceId);
-
- ObjectNode resolveItems(Schema schema, ObjectNode fieldNode, String resourceId);
-
- @Override
- int compare(Schema left, Schema right);
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/schema/SchemaStoreImpl.java b/streams-util/src/main/java/org/apache/streams/util/schema/SchemaStoreImpl.java
deleted file mode 100644
index ddb73c3..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/schema/SchemaStoreImpl.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.schema;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.collect.Ordering;
-import org.apache.commons.lang3.StringUtils;
-import org.jsonschema2pojo.ContentResolver;
-import org.jsonschema2pojo.FragmentResolver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.apache.streams.util.schema.UriUtil.safeResolve;
-
-/**
- * Default Implementation of SchemaStore.
- */
-public class SchemaStoreImpl extends Ordering<Schema> implements SchemaStore {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SchemaStore.class);
- private static final JsonNodeFactory NODE_FACTORY = JsonNodeFactory.instance;
-
- protected Map<URI, Schema> schemas = new HashMap<>();
- protected FragmentResolver fragmentResolver = new FragmentResolver();
- protected ContentResolver contentResolver = new ContentResolver();
-
- public SchemaStoreImpl() {
- }
-
- @Override
- public synchronized Schema create(URI uri) {
- if (!getByUri(uri).isPresent()) {
- URI baseUri = UriUtil.removeFragment(uri);
- JsonNode baseNode = this.contentResolver.resolve(baseUri);
- if (uri.toString().contains("#") && !uri.toString().endsWith("#")) {
- Schema newSchema = new Schema(baseUri, baseNode, null, true);
- this.schemas.put(baseUri, newSchema);
- JsonNode childContent = this.fragmentResolver.resolve(baseNode, '#' + StringUtils.substringAfter(uri.toString(), "#"));
- this.schemas.put(uri, new Schema(uri, childContent, newSchema, false));
- } else {
- if ( baseNode.has("extends") && baseNode.get("extends").isObject()) {
- URI ref = URI.create((baseNode.get("extends")).get("$ref").asText());
- URI absoluteUri;
- if ( ref.isAbsolute()) {
- absoluteUri = ref;
- } else {
- absoluteUri = baseUri.resolve(ref);
- }
- JsonNode parentNode = this.contentResolver.resolve(absoluteUri);
- Schema parentSchema;
- if ( this.schemas.get(absoluteUri) != null ) {
- parentSchema = this.schemas.get(absoluteUri);
- } else {
- parentSchema = create(absoluteUri);
- }
- this.schemas.put(uri, new Schema(uri, baseNode, parentSchema, true));
- } else {
- this.schemas.put(uri, new Schema(uri, baseNode, null, true));
- }
- }
- List<JsonNode> refs = baseNode.findValues("$ref");
- for ( JsonNode ref : refs ) {
- if ( ref.isValueNode() ) {
- String refVal = ref.asText();
- URI refUri = null;
- try {
- refUri = URI.create(refVal);
- } catch ( Exception ex ) {
- LOGGER.info("Exception: {}", ex.getMessage());
- }
- if (refUri != null && !getByUri(refUri).isPresent()) {
- if (refUri.isAbsolute()) {
- create(refUri);
- } else {
- create(baseUri.resolve(refUri));
- }
- }
- }
- }
- }
-
- return this.schemas.get(uri);
- }
-
- @Override
- public Schema create(Schema parent, String path) {
- if (path.equals("#")) {
- return parent;
- } else {
- path = StringUtils.stripEnd(path, "#?&/");
- URI id = (parent != null && parent.getId() != null)
- ? parent.getId().resolve(path)
- : URI.create(path);
- if (this.selfReferenceWithoutParentFile(parent, path)) {
- this.schemas.put(id, new Schema(id, this.fragmentResolver.resolve(parent.getParentContent(), path), parent, false));
- return this.schemas.get(id);
- } else {
- return this.create(id);
- }
- }
- }
-
- protected boolean selfReferenceWithoutParentFile(Schema parent, String path) {
- return parent != null && (parent.getId() == null || parent.getId().toString().startsWith("#/")) && path.startsWith("#/");
- }
-
- @Override
- public synchronized void clearCache() {
- this.schemas.clear();
- }
-
- @Override
- public Integer getSize() {
- return schemas.size();
- }
-
- @Override
- public Optional<Schema> getById(URI id) {
- for ( Schema schema : schemas.values() ) {
- if ( schema.getId() != null && schema.getId().equals(id) ) {
- return Optional.of(schema);
- }
- }
- return Optional.empty();
- }
-
- @Override
- public Optional<Schema> getByUri(URI uri) {
- for ( Schema schema : schemas.values() ) {
- if ( schema.getUri().equals(uri) ) {
- return Optional.of(schema);
- }
- }
- return Optional.empty();
- }
-
- @Override
- public Integer getFileUriCount() {
- int count = 0;
- for ( Schema schema : schemas.values() ) {
- if ( schema.getUri().getScheme().equals("file") ) {
- count++;
- }
- }
- return count;
- }
-
- @Override
- public Integer getHttpUriCount() {
- int count = 0;
- for ( Schema schema : schemas.values() ) {
- if ( schema.getUri().getScheme().equals("http") ) {
- count++;
- }
- }
- return count;
- }
-
- @Override
- public Iterator<Schema> getSchemaIterator() {
- List<Schema> schemaList = new ArrayList<>(schemas.values());
- schemaList.sort(this);
- return schemaList.iterator();
- }
-
- @Override
- public ObjectNode resolveProperties(Schema schema, ObjectNode fieldNode, String resourceId) {
- // this should return something more suitable like:
- // Map<String, Pair<Schema, ObjectNode>>
- ObjectNode schemaProperties = NODE_FACTORY.objectNode();
- ObjectNode parentProperties = NODE_FACTORY.objectNode();
- if (fieldNode == null) {
- ObjectNode schemaContent = (ObjectNode) schema.getContent();
- if (schemaContent.has("properties")) {
- schemaProperties = (ObjectNode) schemaContent.get("properties");
- if (schema.getParentContent() != null) {
- ObjectNode parentContent = (ObjectNode) schema.getParentContent();
- if (parentContent.has("properties")) {
- parentProperties = (ObjectNode) parentContent.get("properties");
- }
- }
- }
- } else if (fieldNode.size() > 0) {
- if (fieldNode.has("properties") && fieldNode.get("properties").isObject() && fieldNode.get("properties").size() > 0) {
- schemaProperties = (ObjectNode) fieldNode.get("properties");
- }
- URI parentUri = null;
- if ( fieldNode.has("$ref") || fieldNode.has("extends") ) {
- JsonNode refNode = fieldNode.get("$ref");
- JsonNode extendsNode = fieldNode.get("extends");
- if (refNode != null && refNode.isValueNode()) {
- parentUri = URI.create(refNode.asText());
- } else if (extendsNode != null && extendsNode.isObject()) {
- parentUri = URI.create(extendsNode.get("$ref").asText());
- }
- ObjectNode parentContent = null;
- URI absoluteUri;
- if (parentUri.isAbsolute()) {
- absoluteUri = parentUri;
- } else {
- absoluteUri = schema.getUri().resolve(parentUri);
- if (!absoluteUri.isAbsolute() || (absoluteUri.isAbsolute() && !getByUri(absoluteUri).isPresent() )) {
- absoluteUri = schema.getParentUri().resolve(parentUri);
- }
- }
- if (absoluteUri.isAbsolute()) {
- if (getByUri(absoluteUri).isPresent()) {
- parentContent = (ObjectNode) getByUri(absoluteUri).get().getContent();
- }
- if (parentContent != null && parentContent.isObject() && parentContent.has("properties")) {
- parentProperties = (ObjectNode) parentContent.get("properties");
- } else if (absoluteUri.getPath().endsWith("#properties")) {
- absoluteUri = URI.create(absoluteUri.toString().replace("#properties", ""));
- parentProperties = (ObjectNode) getByUri(absoluteUri).get().getContent().get("properties");
- }
- }
- }
-
-
- }
-
- ObjectNode resolvedProperties = NODE_FACTORY.objectNode();
- if (parentProperties != null && parentProperties.size() > 0) {
- resolvedProperties = SchemaUtil.mergeProperties(schemaProperties, parentProperties);
- } else {
- resolvedProperties = schemaProperties.deepCopy();
- }
-
- return resolvedProperties;
- }
-
- /**
- * resolve full definition of 'items'.
- * @param schema Schema
- * @param fieldNode ObjectNode
- * @param resourceId resourceId
- * @return ObjectNode
- */
- public ObjectNode resolveItems(Schema schema, ObjectNode fieldNode, String resourceId) {
- ObjectNode schemaItems = NODE_FACTORY.objectNode();
- ObjectNode parentItems = NODE_FACTORY.objectNode();
- if (fieldNode == null) {
- ObjectNode schemaContent = (ObjectNode) schema.getContent();
- if ( schemaContent.has("items") ) {
- schemaItems = (ObjectNode) schemaContent.get("items");
- if (schema.getParentContent() != null) {
- ObjectNode parentContent = (ObjectNode) schema.getParentContent();
- if (parentContent.has("items")) {
- parentItems = (ObjectNode) parentContent.get("items");
- }
- }
- }
- } else if (fieldNode.size() > 0) {
- if (fieldNode.has("items") && fieldNode.get("items").isObject() && fieldNode.get("items").size() > 0) {
- schemaItems = (ObjectNode) fieldNode.get("items");
- }
- URI parentUri = null;
- if ( fieldNode.has("$ref") || fieldNode.has("extends") ) {
- JsonNode refNode = fieldNode.get("$ref");
- JsonNode extendsNode = fieldNode.get("extends");
- if (refNode != null && refNode.isValueNode()) {
- parentUri = URI.create(refNode.asText());
- } else if (extendsNode != null && extendsNode.isObject()) {
- parentUri = URI.create(extendsNode.get("$ref").asText());
- }
- ObjectNode parentContent = null;
- URI absoluteUri;
- if (parentUri.isAbsolute()) {
- absoluteUri = parentUri;
- } else {
- absoluteUri = schema.getUri().resolve(parentUri);
- if (!absoluteUri.isAbsolute() || (absoluteUri.isAbsolute() && !getByUri(absoluteUri).isPresent() )) {
- absoluteUri = schema.getParentUri().resolve(parentUri);
- }
- }
- if (absoluteUri.isAbsolute()) {
- if (getByUri(absoluteUri).isPresent()) {
- parentContent = (ObjectNode) getByUri(absoluteUri).get().getContent();
- }
- if (parentContent != null && parentContent.isObject() && parentContent.has("items")) {
- parentItems = (ObjectNode) parentContent.get("items");
- } else if (absoluteUri.getPath().endsWith("#items")) {
- absoluteUri = URI.create(absoluteUri.toString().replace("#items", ""));
- parentItems = (ObjectNode) getByUri(absoluteUri).get().getContent().get("items");
- }
- }
- }
- }
-
- ObjectNode resolvedItems = NODE_FACTORY.objectNode();
- if (parentItems != null && parentItems.size() > 0) {
- resolvedItems = SchemaUtil.mergeProperties(schemaItems, parentItems);
- } else {
- resolvedItems = schemaItems.deepCopy();
- }
-
- return resolvedItems;
- }
-
- @Override
- public int compare(Schema left, Schema right) {
- // are they the same?
- if ( left.equals(right)) {
- return 0;
- }
- // is one an ancestor of the other
- Schema candidateAncestor = left;
- while ( candidateAncestor.getParent() != null ) {
- candidateAncestor = candidateAncestor.getParent();
- if ( candidateAncestor.equals(right)) {
- return 1;
- }
- }
- candidateAncestor = right;
- while ( candidateAncestor.getParent() != null ) {
- candidateAncestor = candidateAncestor.getParent();
- if ( candidateAncestor.equals(left)) {
- return -1;
- }
- }
- // does one have a field that reference the other?
- for ( JsonNode refNode : left.getContent().findValues("$ref") ) {
- String refText = refNode.asText();
- Optional<URI> resolvedUri = safeResolve(left.getUri(), refText);
- if ( resolvedUri.isPresent() && resolvedUri.get().equals(right.getUri())) {
- return 1;
- }
- }
- for ( JsonNode refNode : right.getContent().findValues("$ref") ) {
- String refText = refNode.asText();
- Optional<URI> resolvedUri = safeResolve(right.getUri(), refText);
- if ( resolvedUri.isPresent() && resolvedUri.get().equals(left.getUri())) {
- return -1;
- }
- }
- // does one have a field that reference a third schema that references the other?
- for ( JsonNode refNode : left.getContent().findValues("$ref") ) {
- String refText = refNode.asText();
- Optional<URI> possibleConnectorUri = safeResolve(left.getUri(), refText);
- if ( possibleConnectorUri.isPresent()) {
- Optional<Schema> possibleConnector = getByUri(possibleConnectorUri.get());
- if (possibleConnector.isPresent()) {
- for (JsonNode connectorRefNode : possibleConnector.get().getContent().findValues("$ref")) {
- String connectorRefText = connectorRefNode.asText();
- Optional<URI> resolvedUri = safeResolve(possibleConnector.get().getUri(), connectorRefText);
- if (resolvedUri.isPresent() && resolvedUri.get().equals(right.getUri())) {
- return 1;
- }
- }
- }
- }
- }
- for ( JsonNode refNode : right.getContent().findValues("$ref") ) {
- String refText = refNode.asText();
- Optional<URI> possibleConnectorUri = safeResolve(right.getUri(), refText);
- if ( possibleConnectorUri.isPresent()) {
- Optional<Schema> possibleConnector = getByUri(possibleConnectorUri.get());
- if (possibleConnector.isPresent()) {
- for (JsonNode connectorRefNode : possibleConnector.get().getContent().findValues("$ref")) {
- String connectorRefText = connectorRefNode.asText();
- Optional<URI> resolvedUri = safeResolve(possibleConnector.get().getUri(), connectorRefText);
- if (resolvedUri.isPresent() && resolvedUri.get().equals(left.getUri())) {
- return -1;
- }
- }
- }
- }
- }
- // there still has to be some order even when there are no connections.
- // we'll arbitrarily pick alphabetic by ID
- int lexigraphic = right.toString().compareTo(left.toString());
- return ( lexigraphic / Math.abs(lexigraphic) );
- }
-
-}
-
diff --git a/streams-util/src/main/java/org/apache/streams/util/schema/SchemaUtil.java b/streams-util/src/main/java/org/apache/streams/util/schema/SchemaUtil.java
deleted file mode 100644
index e4b7928..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/schema/SchemaUtil.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.schema;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URL;
-import java.util.Iterator;
-import java.util.Map;
-
-import static org.apache.commons.lang3.StringUtils.isEmpty;
-
-/**
- * SchemaUtil contains methods to assist in resolving schemas and schema fragments.
- */
-public class SchemaUtil {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SchemaUtil.class);
- private static final JsonNodeFactory NODE_FACTORY = JsonNodeFactory.instance;
- public static final String ILLEGAL_CHARACTER_REGEX = "[^0-9a-zA-Z_$]";
-
- public static String childQualifiedName(String parentQualifiedName, String childSimpleName) {
- String safeChildName = childSimpleName.replaceAll(ILLEGAL_CHARACTER_REGEX, "_");
- return isEmpty(parentQualifiedName) ? safeChildName : parentQualifiedName + "." + safeChildName;
- }
-
- /**
- * read Schema from URL.
- * @param schemaUrl URL
- * @return ObjectNode
- */
- public static ObjectNode readSchema(URL schemaUrl) {
-
- ObjectNode schemaNode = NODE_FACTORY.objectNode();
- schemaNode.put("$ref", schemaUrl.toString());
- return schemaNode;
-
- }
-
- /**
- * merge parent and child properties maps.
- * @param content ObjectNode
- * @param parent ObjectNode
- * @return merged ObjectNode
- */
- public static ObjectNode mergeProperties(ObjectNode content, ObjectNode parent) {
-
- ObjectNode merged = parent.deepCopy();
- Iterator<Map.Entry<String, JsonNode>> fields = content.fields();
- for ( ; fields.hasNext(); ) {
- Map.Entry<String, JsonNode> field = fields.next();
- String fieldId = field.getKey();
- merged.put(fieldId, field.getValue().deepCopy());
- }
- return merged;
- }
-
-}
diff --git a/streams-util/src/main/java/org/apache/streams/util/schema/UriUtil.java b/streams-util/src/main/java/org/apache/streams/util/schema/UriUtil.java
deleted file mode 100644
index 5bd1dc5..0000000
--- a/streams-util/src/main/java/org/apache/streams/util/schema/UriUtil.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.schema;
-
-import org.apache.commons.lang3.StringUtils;
-
-import java.net.URI;
-import java.util.Optional;
-
-/**
- * UriUtil contains methods to assist in resolving URIs and URI fragments.
- */
-public class UriUtil {
-
- public static URI removeFragment(URI id) {
- return URI.create(StringUtils.substringBefore(id.toString(), "#"));
- }
-
- public static URI removeFile(URI id) {
- return URI.create(StringUtils.substringBeforeLast(id.toString(), "/"));
- }
-
- /**
- * resolve a remote schema safely.
- * @param absolute root URI
- * @param relativePart relative to root
- * @return URI if resolvable, or Optional.absent()
- */
- public static Optional<URI> safeResolve(URI absolute, String relativePart) {
- if ( !absolute.isAbsolute()) {
- return Optional.empty();
- }
- try {
- return Optional.of(absolute.resolve(relativePart));
- } catch ( IllegalArgumentException ex ) {
- return Optional.empty();
- }
- }
-
-}
diff --git a/streams-util/src/test/java/org/apache/streams/util/api/requests/backoff/BackOffStrategyTest.java b/streams-util/src/test/java/org/apache/streams/util/api/requests/backoff/BackOffStrategyTest.java
deleted file mode 100644
index 00380e6..0000000
--- a/streams-util/src/test/java/org/apache/streams/util/api/requests/backoff/BackOffStrategyTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.api.requests.backoff;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Unit Test for BackOffStrategy.
- */
-public class BackOffStrategyTest {
-
- private class TestBackOff extends AbstractBackOffStrategy {
-
- public TestBackOff(long sleep, int maxAttempts) {
- super(sleep, maxAttempts);
- }
-
- @Override
- protected long calculateBackOffTime(int attemptCount, long baseSleepTime) {
- return baseSleepTime;
- }
- }
-
- @Test
- public void testUnlimitedBackOff() {
- AbstractBackOffStrategy backOff = new TestBackOff(1, -1);
- try {
- for (int i = 0; i < 100; ++i) {
- backOff.backOff();
- }
- } catch (BackOffException boe) {
- fail("Threw BackOffException. Not expected action");
- }
- }
-
- @Test
- public void testLimitedUseBackOff() {
- AbstractBackOffStrategy backOff = new TestBackOff(1, 2);
- try {
- backOff.backOff();
- } catch (BackOffException boe) {
- fail("Threw BackOffExpection. Not expected action");
- }
- try {
- backOff.backOff();
- } catch (BackOffException boe) {
- fail("Threw BackOffExpection. Not expected action");
- }
- try {
- backOff.backOff();
- fail("Expected BackOffException to be thrown.");
- } catch (BackOffException boe) {
- //
- }
- }
-
- @Test
- public void testBackOffSleep() throws BackOffException {
- AbstractBackOffStrategy backOff = new TestBackOff(2000, 1);
- long startTime = System.currentTimeMillis();
- backOff.backOff();
- long endTime = System.currentTimeMillis();
- assertTrue(endTime - startTime >= 2000);
- }
-
-
-
-}
diff --git a/streams-util/src/test/java/org/apache/streams/util/api/requests/backoff/ConstantTimeBackOffStrategyTest.java b/streams-util/src/test/java/org/apache/streams/util/api/requests/backoff/ConstantTimeBackOffStrategyTest.java
deleted file mode 100644
index 0595235..0000000
--- a/streams-util/src/test/java/org/apache/streams/util/api/requests/backoff/ConstantTimeBackOffStrategyTest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.api.requests.backoff;
-
-import org.apache.streams.util.api.requests.backoff.impl.ConstantTimeBackOffStrategy;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Unit Test for BackOffStrategy.
- */
-public class ConstantTimeBackOffStrategyTest extends RandomizedTest {
-
- @Test
- public void constantTimeBackOffStategy() {
- AbstractBackOffStrategy backOff = new ConstantTimeBackOffStrategy(1);
- Assert.assertEquals(1, backOff.calculateBackOffTime(1,1));
- Assert.assertEquals(1, backOff.calculateBackOffTime(2,1));
- Assert.assertEquals(1, backOff.calculateBackOffTime(3,1));
- Assert.assertEquals(1, backOff.calculateBackOffTime(4,1));
- Assert.assertEquals(1, backOff.calculateBackOffTime(randomIntBetween(1, Integer.MAX_VALUE),1));
- }
-
-
-}
diff --git a/streams-util/src/test/java/org/apache/streams/util/api/requests/backoff/ExponentialBackOffStrategyTest.java b/streams-util/src/test/java/org/apache/streams/util/api/requests/backoff/ExponentialBackOffStrategyTest.java
deleted file mode 100644
index 70f25ec..0000000
--- a/streams-util/src/test/java/org/apache/streams/util/api/requests/backoff/ExponentialBackOffStrategyTest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.api.requests.backoff;
-
-import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Unit Test for ExponentialBackOffStrategy.
- */
-public class ExponentialBackOffStrategyTest {
-
- @Test
- public void exponentialTimeBackOffStrategyTest() {
- AbstractBackOffStrategy backOff = new ExponentialBackOffStrategy(1);
- assertEquals(5000, backOff.calculateBackOffTime(1,5));
- assertEquals(25000, backOff.calculateBackOffTime(2,5));
- assertEquals(125000, backOff.calculateBackOffTime(3,5));
- assertEquals(2000, backOff.calculateBackOffTime(1,2));
- assertEquals(16000, backOff.calculateBackOffTime(4,2));
- }
-
-}
diff --git a/streams-util/src/test/java/org/apache/streams/util/api/requests/backoff/LinearTimeBackOffStrategyTest.java b/streams-util/src/test/java/org/apache/streams/util/api/requests/backoff/LinearTimeBackOffStrategyTest.java
deleted file mode 100644
index 9477b64..0000000
--- a/streams-util/src/test/java/org/apache/streams/util/api/requests/backoff/LinearTimeBackOffStrategyTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.api.requests.backoff;
-
-import org.apache.streams.util.api.requests.backoff.impl.LinearTimeBackOffStrategy;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Unit Test for LinearTimeBackOffStrategy.
- */
-public class LinearTimeBackOffStrategyTest {
-
- @Test
- public void linearTimeBackOffStrategyTest() {
- AbstractBackOffStrategy backOff = new LinearTimeBackOffStrategy(1);
- assertEquals(1000, backOff.calculateBackOffTime(1,1));
- assertEquals(2000, backOff.calculateBackOffTime(2,1));
- assertEquals(3000, backOff.calculateBackOffTime(3,1));
- assertEquals(4000, backOff.calculateBackOffTime(4,1));
- assertEquals(25000, backOff.calculateBackOffTime(5,5));
- }
-}
diff --git a/streams-util/src/test/java/org/apache/streams/util/files/StreamsScannerUtil.java b/streams-util/src/test/java/org/apache/streams/util/files/StreamsScannerUtil.java
deleted file mode 100644
index 97aafd7..0000000
--- a/streams-util/src/test/java/org/apache/streams/util/files/StreamsScannerUtil.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.files;
-
-import java.io.InputStream;
-import java.util.Scanner;
-import java.util.regex.Pattern;
-
-/**
- * Test Utility for acquiring a Scanner that won't choke on unicode or odd line-breaks.
- */
-public class StreamsScannerUtil {
-
- protected static Pattern newLinePattern = Pattern.compile("(\\r\\n?|\\n)", Pattern.MULTILINE);
-
- /**
- * get instance of Scanner using resource path.
- * @param resourcePath resourcePath
- * @return Scanner
- */
- public static Scanner getInstance(String resourcePath) {
-
- InputStream testFileStream = StreamsScannerUtil.class.getResourceAsStream(resourcePath);
- return new Scanner(testFileStream, "UTF-8").useDelimiter(newLinePattern);
-
- }
-}
diff --git a/streams-util/src/test/java/org/apache/streams/util/oauth/tokens/tokenmanager/TestBasicTokenManager.java b/streams-util/src/test/java/org/apache/streams/util/oauth/tokens/tokenmanager/TestBasicTokenManager.java
deleted file mode 100644
index b799fce..0000000
--- a/streams-util/src/test/java/org/apache/streams/util/oauth/tokens/tokenmanager/TestBasicTokenManager.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.oauth.tokens.tokenmanager;
-
-import org.apache.streams.util.oauth.tokens.AbstractOauthToken;
-import org.apache.streams.util.oauth.tokens.tokenmanager.impl.BasicTokenManager;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Unit tests for BasicTokenManager.
- */
-public class TestBasicTokenManager {
-
- /**
- * Simple token for testing purposes.
- */
- private class TestToken extends AbstractOauthToken {
-
- private String token;
-
- public TestToken(String token) {
- this.token = token;
- }
-
- @Override
- protected boolean internalEquals(Object otherToken) {
- if (!(otherToken instanceof TestToken)) {
- return false;
- }
- TestToken that = (TestToken) otherToken;
- return this.token.equals(that.token);
- }
- }
-
- @Test
- public void testNoArgConstructor() {
- try {
- BasicTokenManager manager = new BasicTokenManager<TestToken>();
- assertEquals(0, manager.numAvailableTokens());
- } catch (Throwable throwable) {
- fail("Constructors threw error: " + throwable.getMessage());
- }
- }
-
- @Test
- public void testCollectionConstructor() {
- List<TestToken> tokens = new LinkedList<TestToken>();
- try {
- BasicTokenManager manager1 = new BasicTokenManager<TestToken>(tokens);
- tokens.add(new TestToken("a"));
- tokens.add(new TestToken("b"));
- assertEquals(0, manager1.numAvailableTokens());
- BasicTokenManager manager2 = new BasicTokenManager<TestToken>(tokens);
- assertEquals(2, manager2.numAvailableTokens());
- assertEquals(0, manager1.numAvailableTokens());
- } catch (Throwable throwable) {
- fail("Constructors threw error: " + throwable.getMessage());
- }
- }
-
- @Test
- public void testAddTokenToPool() {
- BasicTokenManager<TestToken> manager = new BasicTokenManager<TestToken>();
- assertTrue(manager.addTokenToPool(new TestToken("a")));
- assertEquals(1, manager.numAvailableTokens());
- assertFalse(manager.addTokenToPool(new TestToken("a")));
- assertEquals(1, manager.numAvailableTokens());
- assertTrue(manager.addTokenToPool(new TestToken("b")));
- assertEquals(2, manager.numAvailableTokens());
- }
-
- @Test
- public void testAddAllTokensToPool() {
- List<TestToken> tokens = new ArrayList<TestToken>();
- tokens.add(new TestToken("a"));
- tokens.add(new TestToken("b"));
- tokens.add(new TestToken("c"));
- BasicTokenManager<TestToken> manager = new BasicTokenManager<TestToken>();
- assertTrue(manager.addAllTokensToPool(tokens));
- assertEquals(3, manager.numAvailableTokens());
- assertFalse(manager.addAllTokensToPool(tokens));
- assertEquals(3, manager.numAvailableTokens());
- tokens.add(new TestToken("d"));
- assertTrue(manager.addAllTokensToPool(tokens));
- assertEquals(4, manager.numAvailableTokens());
- }
-
- @Test
- public void testGetNextAvailableToken() {
- BasicTokenManager manager = new BasicTokenManager<TestToken>();
- assertNull(manager.getNextAvailableToken());
- TestToken tokenA = new TestToken("a");
- assertTrue(manager.addTokenToPool(tokenA));
- assertEquals(tokenA, manager.getNextAvailableToken());
- assertEquals(tokenA, manager.getNextAvailableToken());
- assertEquals(tokenA, manager.getNextAvailableToken());
-
- TestToken tokenB = new TestToken("b");
- TestToken tokenC = new TestToken("c");
- assertTrue(manager.addTokenToPool(tokenB));
- assertTrue(manager.addTokenToPool(tokenC));
- assertEquals(tokenA, manager.getNextAvailableToken());
- assertEquals(tokenB, manager.getNextAvailableToken());
- assertEquals(tokenC, manager.getNextAvailableToken());
- assertEquals(tokenA, manager.getNextAvailableToken());
- assertEquals(tokenB, manager.getNextAvailableToken());
- assertEquals(tokenC, manager.getNextAvailableToken());
- }
-
- @Test
- public void testMultiThreadSafety() {
- int numThreads = 10;
- ExecutorService executor = Executors.newFixedThreadPool(numThreads);
- CountDownLatch startLatch = new CountDownLatch(1);
- CountDownLatch finishLatch = new CountDownLatch(numThreads);
- BasicTokenManager<TestToken> manager = new BasicTokenManager<TestToken>();
- for (int i = 0; i < numThreads; ++i) {
- assertTrue(manager.addTokenToPool(new TestToken(String.valueOf(i))));
- }
- for (int i = 0; i < numThreads; ++i) {
- executor.submit(new TestThread(manager, startLatch, finishLatch, numThreads));
- }
- try {
- Thread.sleep(2000); //sleep for 2 seconds so other threads can initialize
- startLatch.countDown();
- finishLatch.await();
- assertTrue("No errors were thrown during thead safe check", true);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- } catch (Throwable throwable) {
- fail("Error occured durring thread safe test : " + throwable.getMessage());
- }
- }
-
- /**
- * Test class for thread safe check.
- */
- private class TestThread implements Runnable {
-
- private BasicTokenManager<TestToken> manager;
- private CountDownLatch startLatch;
- private CountDownLatch finishedLatch;
- private int availableTokens;
-
- public TestThread(BasicTokenManager<TestToken> manager, CountDownLatch startLatch, CountDownLatch finishedLatch, int availableTokens) {
- this.manager = manager;
- this.startLatch = startLatch;
- this.finishedLatch = finishedLatch;
- this.availableTokens = availableTokens;
- }
-
- @Override
- public void run() {
- try {
- this.startLatch.await();
- for (int i = 0; i < 1000; ++i) {
- assertNotNull(this.manager.getNextAvailableToken());
- assertEquals(this.availableTokens, this.manager.numAvailableTokens());
- }
- this.finishedLatch.countDown();
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- } catch (Throwable throwable) {
- fail("Threw error in multithread test : " + throwable.getMessage());
- }
- }
- }
-
-}
diff --git a/streams-util/src/test/java/org/apache/streams/util/schema/test/PropertyUtilTest.java b/streams-util/src/test/java/org/apache/streams/util/schema/test/PropertyUtilTest.java
deleted file mode 100644
index d322247..0000000
--- a/streams-util/src/test/java/org/apache/streams/util/schema/test/PropertyUtilTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.schema.test;
-
-import org.apache.streams.util.PropertyUtil;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.junit.Test;
-
-/**
- * Unit Test for PropertyUtil.
- */
-public class PropertyUtilTest {
-
- private static final ObjectMapper mapper = new ObjectMapper();
-
- String flatJson = "{\"a.b\": \"ab\", \"c.d\": \"cd\", \"a.e\": \"ae\", \"c.f\": \"cf\"}";
-
- @Test
- public void testUnflattenObjectNode() throws Exception {
- PropertyUtil propertyUtil = PropertyUtil.getInstance();
- ObjectNode flatNode = mapper.readValue(flatJson, ObjectNode.class);
- ObjectNode unflattenedNode = propertyUtil.unflattenObjectNode(flatNode);
- assert(unflattenedNode.size() == 2);
- assert(unflattenedNode.get("a") != null);
- assert(unflattenedNode.get("b") == null);
- assert(unflattenedNode.get("c") != null);
- assert(unflattenedNode.get("a").size() == 2);
- assert(unflattenedNode.get("a").get("b").asText().equals("ab"));
- assert(unflattenedNode.get("a").get("e").asText().equals("ae"));
- assert(unflattenedNode.get("c").size() == 2);
- assert(unflattenedNode.get("c").get("d").asText().equals("cd"));
- assert(unflattenedNode.get("c").get("f").asText().equals("cf"));
- }
-}
-
diff --git a/streams-util/src/test/java/org/apache/streams/util/schema/test/SchemaOrderingTest.java b/streams-util/src/test/java/org/apache/streams/util/schema/test/SchemaOrderingTest.java
deleted file mode 100644
index 7c574b7..0000000
--- a/streams-util/src/test/java/org/apache/streams/util/schema/test/SchemaOrderingTest.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.schema.test;
-
-import org.apache.streams.util.schema.Schema;
-import org.apache.streams.util.schema.SchemaStore;
-import org.apache.streams.util.schema.SchemaStoreImpl;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterators;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Unit Tests for SchemaOrdering.
- */
-public class SchemaOrderingTest {
-
- @Test
- public void compareVerbParent() {
- SchemaStore schemaStore = new SchemaStoreImpl();
- File update = new File("target/test-classes/activitystreams-schemas/verbs/update.json");
- schemaStore.create(update.toURI());
- File activity = new File("target/test-classes/activitystreams-schemas/activity.json");
- schemaStore.create(activity.toURI());
- assert (schemaStore.compare( schemaStore.getByUri(update.toURI()).get(), schemaStore.getByUri(activity.toURI()).get()) == 1);
- Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator();
- assertContainsItemsEndingWithInOrder(
- schemaIterator,
- Arrays.asList(
- "activity.json",
- "update.json"
- )
- );
- }
-
- @Test
- public void compareObjectTypeParent() {
- SchemaStore schemaStore = new SchemaStoreImpl();
- File alert = new File("target/test-classes/activitystreams-schemas/objectTypes/alert.json");
- schemaStore.create(alert.toURI());
- File object = new File("target/test-classes/activitystreams-schemas/object.json");
- schemaStore.create(object.toURI());
- assert (schemaStore.compare( schemaStore.getByUri(object.toURI()).get(), schemaStore.getByUri(alert.toURI()).get()) == -1);
- Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator();
- assertContainsItemsEndingWithInOrder(
- schemaIterator,
- Arrays.asList(
- "object.json",
- "alert.json"
- )
- );
- }
-
- @Test
- public void compareUnrelated() {
- SchemaStore schemaStore = new SchemaStoreImpl();
- File alert = new File("target/test-classes/activitystreams-schemas/objectTypes/alert.json");
- schemaStore.create(alert.toURI());
- File update = new File("target/test-classes/activitystreams-schemas/verbs/update.json");
- schemaStore.create(update.toURI());
- // update > alert b/c it's ascii is higher alphabetically
- assert (schemaStore.compare( schemaStore.getByUri(alert.toURI()).get(), schemaStore.getByUri(update.toURI()).get()) != 0);
- }
-
- @Test
- public void compareVerbFieldRef() {
- SchemaStore schemaStore = new SchemaStoreImpl();
- File update = new File("target/test-classes/activitystreams-schemas/verbs/update.json");
- schemaStore.create(update.toURI());
- File object = new File("target/test-classes/activitystreams-schemas/object.json");
- schemaStore.create(object.toURI());
- assert ( schemaStore.compare( schemaStore.getByUri(update.toURI()).get(), schemaStore.getByUri(object.toURI()).get()) == 1);
- Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator();
- assertContainsItemsEndingWithInOrder(
- schemaIterator,
- Arrays.asList(
- "object.json",
- "update.json"
- )
- );
- }
-
- @Test
- public void compareObjectTypeFieldRef() {
- SchemaStore schemaStore = new SchemaStoreImpl();
- File alert = new File("target/test-classes/activitystreams-schemas/objectTypes/alert.json");
- schemaStore.create(alert.toURI());
- File mediaLink = new File("target/test-classes/activitystreams-schemas/media_link.json");
- schemaStore.create(mediaLink.toURI());
- assert ( schemaStore.compare( schemaStore.getByUri(mediaLink.toURI()).get(), schemaStore.getByUri(alert.toURI()).get()) == -1);
- Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator();
- assertContainsItemsEndingWithInOrder(
- schemaIterator,
- Arrays.asList(
- "media_link.json",
- "object.json",
- "alert.json"
- )
- );
- }
-
- @Test
- public void compareVerbAncestorIndirect() {
- SchemaStore schemaStore = new SchemaStoreImpl();
- File update = new File("target/test-classes/activitystreams-schemas/verbs/update.json");
- schemaStore.create(update.toURI());
- File mediaLink = new File("target/test-classes/activitystreams-schemas/media_link.json");
- schemaStore.create(mediaLink.toURI());
- Assert.assertTrue(schemaStore.getByUri(mediaLink.toURI()).isPresent());
- Assert.assertTrue(schemaStore.getByUri(update.toURI()).isPresent());
- Assert.assertTrue(schemaStore.compare( schemaStore.getByUri(mediaLink.toURI()).get(),
- schemaStore.getByUri(update.toURI()).get()) == -1);
- Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator();
- assertContainsItemsEndingWithInOrder(
- schemaIterator,
- Arrays.asList(
- "media_link.json",
- "update.json"
- )
- );
- }
-
- /**
- * assert iterator of Schema contains URI items ending with in order.
- * @param iterator Iterator of Schema
- * @param items List of String
- */
- public void assertContainsItemsEndingWithInOrder(Iterator<Schema> iterator, List<String> items) {
- for ( String item : items ) {
- Optional<Schema> tryFind = Iterators.tryFind( iterator, new SchemaUriEndsWithPredicate(item) );
- Assert.assertTrue(tryFind.isPresent());
- }
- }
-
- public class SchemaUriEndsWithPredicate implements Predicate<Schema> {
-
- private String endsWith;
-
- public SchemaUriEndsWithPredicate(String endsWith) {
- this.endsWith = endsWith;
- }
-
- @Override
- public boolean apply(Schema input) {
- return input.getUri().getPath().endsWith(endsWith);
- }
-
- @Override
- public boolean equals(Object object) {
- return false;
- }
- }
-}
diff --git a/streams-util/src/test/java/org/apache/streams/util/schema/test/SchemaStoreTest.java b/streams-util/src/test/java/org/apache/streams/util/schema/test/SchemaStoreTest.java
deleted file mode 100644
index 2746bd7..0000000
--- a/streams-util/src/test/java/org/apache/streams/util/schema/test/SchemaStoreTest.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.util.schema.test;
-
-import org.apache.streams.util.schema.Schema;
-import org.apache.streams.util.schema.SchemaStore;
-import org.apache.streams.util.schema.SchemaStoreImpl;
-
-import org.junit.Test;
-
-import java.io.File;
-
-/**
- * Unit Tests for SchemaStore.
- */
-public class SchemaStoreTest {
-
- @Test
- public void indexMediaLink() {
- SchemaStore schemaStore = new SchemaStoreImpl();
- File file = new File("target/test-classes/activitystreams-schemas/media_link.json");
- schemaStore.create(file.toURI());
- assert ( schemaStore.getFileUriCount() == 1);
- assert ( schemaStore.getByUri(file.toURI()).isPresent());
- assert ( schemaStore.getById(schemaStore.getByUri(file.toURI()).get().getId()).isPresent());
- }
-
- @Test
- public void indexApprove() {
- SchemaStore schemaStore = new SchemaStoreImpl();
- File file = new File("target/test-classes/activitystreams-schemas/verbs/approve.json");
- schemaStore.create(file.toURI());
- assert ( schemaStore.getFileUriCount() == 4);
- assert ( schemaStore.getByUri(file.toURI()).isPresent());
- assert ( schemaStore.getById(schemaStore.getByUri(file.toURI()).get().getId()).isPresent());
- }
-
- @Test
- public void indexCollection() {
- SchemaStore schemaStore = new SchemaStoreImpl();
- File file = new File("target/test-classes/activitystreams-schemas/collection.json");
- schemaStore.create(file.toURI());
- assert ( schemaStore.getFileUriCount() == 3);
- assert ( schemaStore.getByUri(file.toURI()).isPresent());
- assert ( schemaStore.getById(schemaStore.getByUri(file.toURI()).get().getId()).isPresent());
- Schema collection = schemaStore.getByUri(file.toURI()).get();
- assert ( collection.getParent() == null );
- }
-
- @Test
- public void indexUpdate() {
- SchemaStore schemaStore = new SchemaStoreImpl();
- File file = new File("target/test-classes/activitystreams-schemas/verbs/update.json");
- schemaStore.create(file.toURI());
- assert ( schemaStore.getFileUriCount() == 4);
- assert ( schemaStore.getByUri(file.toURI()).isPresent());
- assert ( schemaStore.getById(schemaStore.getByUri(file.toURI()).get().getId()).isPresent());
- Schema update = schemaStore.getByUri(file.toURI()).get();
- assert ( update.getParent() != null );
- File parentFile = new File("target/test-classes/activitystreams-schemas/activity.json");
- Schema parent = schemaStore.getByUri(parentFile.toURI()).get();
- assert ( parent != null );
- assert ( update.getParentUri().equals(parent.getUri()));
- }
-
- // test create from messed up URI
-
- // test create from URI with messed up reference
-
-}
diff --git a/streams-verbs/README.md b/streams-verbs/README.md
deleted file mode 100644
index 6d93b8b..0000000
--- a/streams-verbs/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-Apache Streams (incubating)
-Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
---------------------------------------------------------------------------------
-
-org.apache.streams:streams-util
-===============================
-
-[README.md](src/site/markdown/index.md "README")
diff --git a/streams-verbs/pom.xml b/streams-verbs/pom.xml
deleted file mode 100644
index 32a2166..0000000
--- a/streams-verbs/pom.xml
+++ /dev/null
@@ -1,177 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.streams</groupId>
- <artifactId>apache-streams</artifactId>
- <version>0.5.1-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>streams-verbs</artifactId>
-
- <name>streams-verbs</name>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.datatype</groupId>
- <artifactId>jackson-datatype-joda</artifactId>
- </dependency>
-
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
-
- <dependency>
- <groupId>nz.net.ultraq.jaxb</groupId>
- <artifactId>jaxb-utilities</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.antlr</groupId>
- <artifactId>stringtemplate</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-testing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- </dependencies>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-pojo</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.streams</groupId>
- <artifactId>streams-util</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.antlr</groupId>
- <artifactId>stringtemplate</artifactId>
- <version>4.0.2</version>
- </dependency>
- </dependencies>
-
-
- </dependencyManagement>
-
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <testSourceDirectory>src/test/java</testSourceDirectory>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.streams.plugins</groupId>
- <artifactId>streams-plugin-pojo</artifactId>
- <version>${project.version}</version>
- <configuration>
- <sourcePaths>
- <sourcePath>${project.basedir}/src/main/jsonschema</sourcePath>
- </sourcePaths>
- <targetDirectory>${project.basedir}/target/generated-sources/pojo</targetDirectory>
- <targetPackage>org.apache.streams.verbs</targetPackage>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-sources/pojo</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes combine.children="append">
- <exclude>src/test/resources/activities.txt</exclude>
- <exclude>src/test/resources/actor.json</exclude>
- <exclude>src/test/resources/do.json</exclude>
- <exclude>src/test/resources/follow.json</exclude>
- <exclude>src/test/resources/nofields.json</exclude>
- <exclude>src/test/resources/object.json</exclude>
- <exclude>src/test/resources/post.json</exclude>
- <exclude>src/test/resources/provider.json</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-</project>
diff --git a/streams-verbs/src/main/java/org/apache/streams/verbs/ObjectCombinationGenericOrdering.java b/streams-verbs/src/main/java/org/apache/streams/verbs/ObjectCombinationGenericOrdering.java
deleted file mode 100644
index 228fba2..0000000
--- a/streams-verbs/src/main/java/org/apache/streams/verbs/ObjectCombinationGenericOrdering.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.verbs;
-
-import com.google.common.collect.Ordering;
-
-/**
- * Orders ObjectCombinations from most specific to most general, without regard
- * for degree of match to any specific Activity.
- */
-public class ObjectCombinationGenericOrdering extends Ordering<ObjectCombination> {
-
- public ObjectCombinationGenericOrdering() {}
-
- @Override
- public int compare(ObjectCombination left, ObjectCombination right) {
- if ( wildcardCount(left) < wildcardCount(right)) {
- return -1;
- } else if ( wildcardCount(left) > wildcardCount(right)) {
- return 1;
- } else if ( !wildcard(left.getActor()) && wildcard(right.getActor())) {
- return -1;
- } else if ( wildcard(left.getActor()) && !wildcard(right.getActor())) {
- return 1;
- } else if ( !wildcard(left.getObject()) && wildcard(right.getObject())) {
- return -1;
- } else if ( wildcard(left.getObject()) && !wildcard(right.getObject())) {
- return 1;
- } else if ( !wildcard(left.getTarget()) && wildcard(right.getTarget())) {
- return -1;
- } else if ( wildcard(left.getTarget()) && !wildcard(right.getTarget())) {
- return 1;
- } else if ( !wildcard(left.getProvider()) && wildcard(right.getProvider())) {
- return -1;
- } else if ( wildcard(left.getProvider()) && !wildcard(right.getProvider())) {
- return 1;
- } else {
- return 0;
- }
- }
-
- /**
- * count wildcards in this ObjectCombination.
- * @param objectCombination ObjectCombination
- * @return count
- */
- private int wildcardCount(ObjectCombination objectCombination) {
- int wildcardCount = 0;
- if ( wildcard(objectCombination.getActor())) {
- wildcardCount++;
- }
- if ( wildcard(objectCombination.getObject())) {
- wildcardCount++;
- }
- if ( wildcard(objectCombination.getTarget())) {
- wildcardCount++;
- }
- if ( wildcard(objectCombination.getProvider())) {
- wildcardCount++;
- }
- return wildcardCount;
- }
-
- /**
- * is pattern a wildcard.
- * @param pattern String
- * @return true or false
- */
- private boolean wildcard(String pattern) {
- return pattern.equals("*");
- }
-}
diff --git a/streams-verbs/src/main/java/org/apache/streams/verbs/ObjectCombinationSpecificOrdering.java b/streams-verbs/src/main/java/org/apache/streams/verbs/ObjectCombinationSpecificOrdering.java
deleted file mode 100644
index 8b395bd..0000000
--- a/streams-verbs/src/main/java/org/apache/streams/verbs/ObjectCombinationSpecificOrdering.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.verbs;
-
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import com.google.common.collect.Ordering;
-
-/**
- * Orders ObjectCombinations from most specific to most general, in context of
- * degree of match to a specified Activity.
- */
-public class ObjectCombinationSpecificOrdering extends Ordering<ObjectCombination> {
-
- private Activity activity;
-
- public ObjectCombinationSpecificOrdering(Activity activity) {
- this.activity = activity;
- }
-
- @Override
- public int compare(ObjectCombination left, ObjectCombination right) {
- if (matchCount(left) < matchCount(right)) {
- return 1;
- } else if ( matchCount(left) > matchCount(right)) {
- return -1;
- } else if ( !match(activity.getActor(), left.getActor()) && match(activity.getActor(), right.getActor())) {
- return 1;
- } else if ( match(activity.getActor(), left.getActor()) && !match(activity.getActor(), right.getActor())) {
- return -1;
- } else if ( !match(activity.getObject(), left.getObject()) && match(activity.getObject(), right.getObject())) {
- return 1;
- } else if ( match(activity.getObject(), left.getObject()) && !match(activity.getObject(), right.getObject())) {
- return -1;
- } else if ( !match(activity.getTarget(), left.getTarget()) && match(activity.getTarget(), right.getTarget())) {
- return 1;
- } else if ( match(activity.getTarget(), left.getTarget()) && !match(activity.getTarget(), right.getTarget())) {
- return -1;
- } else if ( !match(activity.getProvider(), left.getProvider()) && match(activity.getTarget(), right.getProvider())) {
- return 1;
- } else if ( match(activity.getProvider(), left.getProvider()) && !match(activity.getTarget(), right.getProvider())) {
- return -1;
- } else {
- return 0;
- }
- }
-
- /**
- * count matches between this ObjectCombination and this Activity.
- * @param objectCombination ObjectCombination
- * @return count
- */
- private int matchCount(ObjectCombination objectCombination) {
- int matchCount = 0;
- if ( match(activity.getActor(), objectCombination.getActor())) {
- matchCount++;
- }
- if ( match(activity.getObject(), objectCombination.getObject())) {
- matchCount++;
- }
- if ( match(activity.getTarget(), objectCombination.getTarget())) {
- matchCount++;
- }
- if ( match(activity.getProvider(), objectCombination.getProvider())) {
- matchCount++;
- }
- return matchCount;
- }
-
- /**
- * whether this ActivityObject matches the corresponding ObjectCombination pattern.
- * @param activityObject ActivityObject
- * @param pattern pattern
- * @return true or false
- */
- public boolean match(ActivityObject activityObject, String pattern) {
- return activityObject != null
- && activityObject.getObjectType() != null
- && activityObject.getObjectType().equals(pattern);
- }
-}
diff --git a/streams-verbs/src/main/java/org/apache/streams/verbs/VerbDefinitionMatchUtil.java b/streams-verbs/src/main/java/org/apache/streams/verbs/VerbDefinitionMatchUtil.java
deleted file mode 100644
index 051be22..0000000
--- a/streams-verbs/src/main/java/org/apache/streams/verbs/VerbDefinitionMatchUtil.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.verbs;
-
-import org.apache.streams.pojo.json.Activity;
-
-import java.util.Set;
-
-/**
- * Check whether an activity matches one or several VerbDefinition.
- */
-public class VerbDefinitionMatchUtil {
-
- /**
- * whether this Activity matches any of a Set of VerbDefinitions.
- * @param activity Activity
- * @param verbDefinitionSet Set of VerbDefinition
- * @return true or false
- */
- public static boolean match(Activity activity, Set<VerbDefinition> verbDefinitionSet) {
-
- for ( VerbDefinition verbDefinition : verbDefinitionSet) {
- if ( match( activity, verbDefinition )) {
- return true;
- }
- }
- return false;
-
- }
-
- /**
- * whether this Activity matches this VerbDefinition.
- * @param activity Activity
- * @param verbDefinition VerbDefinition
- * @return true or false
- */
- public static boolean match(Activity activity, VerbDefinition verbDefinition) {
-
- if ( verbDefinition.getValue() != null
- && verbDefinition.getValue().equals(activity.getVerb())) {
- for (ObjectCombination objectCombination : verbDefinition.getObjects()) {
- if (VerbDefinitionResolver.filter(activity, objectCombination)) {
- return true;
- }
- }
- }
- return false;
- }
-
-}
diff --git a/streams-verbs/src/main/java/org/apache/streams/verbs/VerbDefinitionResolver.java b/streams-verbs/src/main/java/org/apache/streams/verbs/VerbDefinitionResolver.java
deleted file mode 100644
index 023b2f3..0000000
--- a/streams-verbs/src/main/java/org/apache/streams/verbs/VerbDefinitionResolver.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.verbs;
-
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.util.SerializationUtil;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class VerbDefinitionResolver {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(VerbDefinitionResolver.class);
-
- protected Set<VerbDefinition> verbDefinitionSet;
-
- public VerbDefinitionResolver() {
- // get with reflection
- }
-
- public VerbDefinitionResolver(Set<VerbDefinition> verbDefinitionSet) {
- this();
- this.verbDefinitionSet = verbDefinitionSet;
- }
-
- /**
- * return all matching VerbDefinitions for an Activity.
- * @param activity Activity
- * @return List of VerbDefinition
- */
- public Set<VerbDefinition> matchingVerbDefinitions(Activity activity) {
-
- // ConcurrentHashSet is preferable, but it's only in guava 15+
- // spark 1.5.0 uses guava 14 so for the moment this is the workaround
- // Set<VerbDefinition> matches = Sets.newConcurrentHashSet();
- Set<VerbDefinition> matches = Collections.newSetFromMap(new ConcurrentHashMap<VerbDefinition, Boolean>());
-
- for (VerbDefinition verbDefinition : verbDefinitionSet) {
- VerbDefinition verbDefinitionCopy = SerializationUtil.cloneBySerialization(verbDefinition);
- if ( activity.getVerb().equals(verbDefinition.getValue())) {
- for ( ObjectCombination criteria : verbDefinitionCopy.getObjects()) {
- if (!filter(activity, criteria)) {
- verbDefinitionCopy.getObjects().remove(criteria);
- }
- }
- if ( verbDefinitionCopy.getObjects().size() > 0) {
- matches.add(verbDefinitionCopy);
- }
- }
- }
-
- return matches;
-
- }
-
- /**
- * return all matching ObjectCombinations for an Activity.
- * @param activity Activity
- * @return List of ObjectCombination
- */
- public List<ObjectCombination> matchingObjectCombinations(Activity activity) {
-
- List<ObjectCombination> results = new ArrayList<>();
-
- for ( VerbDefinition verbDefinition : verbDefinitionSet ) {
- if ( activity.getVerb().equals(verbDefinition.getValue())) {
- for ( ObjectCombination criteria : verbDefinition.getObjects()) {
- if (filter(activity, criteria)) {
- results.add(criteria);
- }
- }
- }
- }
-
- results.sort(new ObjectCombinationSpecificOrdering(activity));
-
- return results;
- }
-
- /**
- * whether this Activity matches this ObjectCombination.
- * @param activity Activity
- * @param criteria ObjectCombination
- * @return true or false
- */
- public static boolean filter(Activity activity, ObjectCombination criteria) {
-
- return filterType(activity.getActor(), criteria.getActorRequired(), criteria.getActor())
- &&
- filterType(activity.getObject(), criteria.getObjectRequired(), criteria.getObject())
- &&
- filterType(activity.getProvider(), criteria.getProviderRequired(), criteria.getProvider())
- &&
- filterType(activity.getTarget(), criteria.getTargetRequired(), criteria.getTarget())
- ;
-
- }
-
- public static boolean filterType(ActivityObject activityObject, boolean required, String pattern) {
- return !(required && activityObject == null) &&
- (!required && activityObject == null ||
- pattern.equals("*") ||
- activityObject.getObjectType() != null &&
- activityObject.getObjectType().equals(pattern));
- }
-
-
-
-}
diff --git a/streams-verbs/src/main/java/org/apache/streams/verbs/VerbDefinitionTemplateUtil.java b/streams-verbs/src/main/java/org/apache/streams/verbs/VerbDefinitionTemplateUtil.java
deleted file mode 100644
index e34f462..0000000
--- a/streams-verbs/src/main/java/org/apache/streams/verbs/VerbDefinitionTemplateUtil.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.verbs;
-
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-
-import org.apache.commons.lang3.StringUtils;
-import org.stringtemplate.v4.ST;
-
-/**
- * Transforms VerbDefinition templates into readable strings.
- */
-public class VerbDefinitionTemplateUtil {
-
- /**
- * Transform Activity into readable string using ObjectCombination title.
- * @param activity Activity
- * @param objectCombination ObjectCombination
- * @return String
- */
- public static String asString(Activity activity, ObjectCombination objectCombination) {
-
- return asString("*", activity, objectCombination);
-
- }
-
- /**
- * Transform Activity into readable string using ObjectCombination title and specified language.
- * @param language language
- * @param activity Activity
- * @param objectCombination ObjectCombination
- * @return String
- */
- public static String asString(String language, Activity activity, ObjectCombination objectCombination) {
-
- String template = (String) objectCombination.getTemplates().getAdditionalProperties().get(language);
- template = template.replace('{', '<');
- template = template.replace('}', '>');
- ST st = new ST(template);
- st.add("actor", displayName(activity.getActor()));
- st.add("provider", displayName(activity.getProvider()));
- st.add("object", displayName(activity.getObject()));
- st.add("target", displayName(activity.getTarget()));
-
- return st.render();
- }
-
- /**
- * Readable display Name for ActivityObject.
- * @param activityObject ActivityObject
- * @return displayName
- */
- public static String displayName(ActivityObject activityObject) {
- if ( activityObject == null ) {
- return "";
- } else if (StringUtils.isNotBlank(activityObject.getDisplayName())) {
- return activityObject.getDisplayName();
- } else if (StringUtils.isNotBlank(activityObject.getObjectType())) {
- return activityObject.getObjectType();
- } else if (StringUtils.isNotBlank(activityObject.toString())) {
- return activityObject.toString();
- } else {
- return "";
- }
- }
-}
diff --git a/streams-verbs/src/main/jsonschema/verb-definition.json b/streams-verbs/src/main/jsonschema/verb-definition.json
deleted file mode 100644
index 572914d..0000000
--- a/streams-verbs/src/main/jsonschema/verb-definition.json
+++ /dev/null
@@ -1,99 +0,0 @@
-{
- "type" : "object",
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "title" : "activity",
- "javaType": "org.apache.streams.verbs.VerbDefinition",
- "javaInterfaces": ["java.io.Serializable"],
- "description" : "A verb definition declares what objectTypes should accompany a specific verb in an activity",
- "additionalProperties": false,
- "properties": {
- "objectType" : {
- "title" : "objectType",
- "type" : "string",
- "required" : true,
- "default" : "verb",
- "description" :"verb"
- },
- "id" :{
- "type" : "string",
- "description" : "The unique identifier for the verb object",
- "required" : true
- },
- "value" : {
- "title" : "value",
- "type" : "string",
- "required": true,
- "description" :"The specific String that is to be used for the 'verb' property within an Activity object."
- },
- "displayName" : {
- "title" : "displayName",
- "type" : "string",
- "description" :"A natural-language, human-readable and plain-text display name for the verb."
- },
- "synonyms" : {
- "type" : "array",
- "description" :"An array of zero or more other Strings known verbs for which this verb can be considered to be a synonym or alias.",
- "note": "Tell JSON schema team to not put links inside http://json-schema.org/hyper-schema#properties",
- "items": {
- "type" : "string"
- }
- },
- "hypernyms" : {
- "type" : "array",
- "description" :"An array of zero or more other Strings known verbs for which this verb can be considered to be a synonym or alias.",
- "items": {
- "type" : "string"
- }
- },
- "objects" : {
- "type" : "array",
- "description" :"An array of Object Combinations.",
- "items": {
- "type" : "object",
- "javaType" : "org.apache.streams.verbs.ObjectCombination",
- "javaInterfaces": ["java.io.Serializable"],
- "properties": {
- "actor": {
- "type" : "string",
- "default": "*"
- },
- "actorRequired": {
- "type" : "boolean",
- "default": "false"
- },
- "object": {
- "type" : "string",
- "default": "*"
- },
- "objectRequired": {
- "type" : "boolean",
- "default": "false"
- },
- "provider": {
- "type" : "string",
- "default": "*"
- },
- "providerRequired": {
- "type" : "boolean",
- "default": "false"
- },
- "target": {
- "type" : "string",
- "default": "*"
- },
- "targetRequired": {
- "type" : "boolean",
- "default": "false"
- },
- "templates" : {
- "type": "object",
- "javaInterfaces": ["java.io.Serializable"],
- "additionalProperties": true
- }
- }
- }
- }
- }
-}
diff --git a/streams-verbs/src/main/resources/delete.json b/streams-verbs/src/main/resources/delete.json
deleted file mode 100644
index 4dc50ac..0000000
--- a/streams-verbs/src/main/resources/delete.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "objectType": "verb",
- "id": "http://streams.apache.org/verbs/delete",
- "value": "delete",
- "objects": [
- {
- "actor": "page",
- "object": "activity",
- "target": "*",
- "targetRequired": false,
- "templates": {
- "*": "{actor.displayName} deleted {object.displayName}"
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-verbs/src/main/resources/follow.json b/streams-verbs/src/main/resources/follow.json
deleted file mode 100644
index 14cde7e..0000000
--- a/streams-verbs/src/main/resources/follow.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "objectType": "verb",
- "id": "http://streams.apache.org/verbs/follow",
- "value": "follow",
- "objects": [
- {
- "actor": "page",
- "object": "page",
- "target": "*",
- "targetRequired": false,
- "templates": {
- "*": "{actor.displayName} followed {object.displayName}"
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-verbs/src/main/resources/post.json b/streams-verbs/src/main/resources/post.json
deleted file mode 100644
index 83cab81..0000000
--- a/streams-verbs/src/main/resources/post.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "objectType": "verb",
- "id": "http://streams.apache.org/verbs/post",
- "value": "post",
- "objects": [
- {
- "actor": "page",
- "object": "note",
- "target": "*",
- "targetRequired": false,
- "templates": {
- "*": "{actor.displayName} posted {object.content}"
- }
- }
- ]
-}
diff --git a/streams-verbs/src/main/resources/share.json b/streams-verbs/src/main/resources/share.json
deleted file mode 100644
index 3f0497c..0000000
--- a/streams-verbs/src/main/resources/share.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "$license": [
- "http://www.apache.org/licenses/LICENSE-2.0"
- ],
- "objectType": "verb",
- "id": "http://streams.apache.org/verbs/share",
- "value": "share",
- "objects": [
- {
- "actor": "page",
- "object": "activity",
- "target": "*",
- "targetRequired": false,
- "templates": {
- "*": "{actor.displayName} shared {object.content}"
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-verbs/src/site/markdown/index.md b/streams-verbs/src/site/markdown/index.md
deleted file mode 100644
index d79f467..0000000
--- a/streams-verbs/src/site/markdown/index.md
+++ /dev/null
@@ -1,19 +0,0 @@
-streams-verbs
-=================
-
-An "Object Combination" a description of what types of actor, object
-and target objects are typically expected to be used with a
-particular verb.
-
-Classes in streams-verbs perform registration of activity stream verb definitions,
-allowing a stream or component to specify the type of activities deemed valid.
-
-Resources in streams-verbs are a recommended baseline set of verb definitions for
-activities produced by multiple contributed providers, and templates for describing
-those activities.
-
-https://raw.github.com/activitystreams/activity-streams-verb-definition/master/activity-streams-verb-definition.txt
-
-[JavaDocs](apidocs/index.html "JavaDocs")
-
-###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
diff --git a/streams-verbs/src/test/java/org/apache/streams/verbs/ObjectCombinationGenericOrderingTest.java b/streams-verbs/src/test/java/org/apache/streams/verbs/ObjectCombinationGenericOrderingTest.java
deleted file mode 100644
index 16e6167..0000000
--- a/streams-verbs/src/test/java/org/apache/streams/verbs/ObjectCombinationGenericOrderingTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.verbs;
-
-import org.junit.Test;
-
-/**
- * Orders ObjectCombinations from most specific to most general, without regard
- * for degree of match to any specific Activity.
- */
-public class ObjectCombinationGenericOrderingTest {
-
- @Test
- public void compareWildcardCountTest() {
- ObjectCombination combination1 = new ObjectCombination();
- ObjectCombination combination2 = new ObjectCombination().withActor("actor");
- assert (new ObjectCombinationGenericOrdering()).compare(combination1, combination2) > 0;
- ObjectCombination combination3 = new ObjectCombination();
- ObjectCombination combination4 = new ObjectCombination().withProvider("provider");
- assert (new ObjectCombinationGenericOrdering()).compare(combination3, combination4) > 0;
- }
-
-
-}
diff --git a/streams-verbs/src/test/java/org/apache/streams/verbs/ObjectCombinationSpecificOrderingTest.java b/streams-verbs/src/test/java/org/apache/streams/verbs/ObjectCombinationSpecificOrderingTest.java
deleted file mode 100644
index 8162734..0000000
--- a/streams-verbs/src/test/java/org/apache/streams/verbs/ObjectCombinationSpecificOrderingTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.verbs;
-
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Provider;
-
-import org.junit.Test;
-
-/**
- * Orders ObjectCombinations from most specific to most general, without regard
- * for degree of match to any specific Activity.
- */
-public class ObjectCombinationSpecificOrderingTest {
-
- @Test
- public void compareMatchCountTest() {
- ActivityObject actor = new ActivityObject();
- actor.setObjectType("actor");
- Activity activity = new Activity().withActor(actor);
- ObjectCombination combination1 = new ObjectCombination();
- ObjectCombination combination2 = new ObjectCombination().withActor("actor");
- assert (new ObjectCombinationSpecificOrdering(activity)).compare(combination1, combination2) > 0;
- Provider provider = new Provider();
- provider.setObjectType("application");
- Activity activity2 = new Activity().withProvider(provider);
- ObjectCombination combination3 = new ObjectCombination();
- ObjectCombination combination4 = new ObjectCombination().withProvider("application");
- assert (new ObjectCombinationSpecificOrdering(activity2)).compare(combination3, combination4) > 0;
- }
-
-
-}
diff --git a/streams-verbs/src/test/java/org/apache/streams/verbs/VerbDefinitionResolverTest.java b/streams-verbs/src/test/java/org/apache/streams/verbs/VerbDefinitionResolverTest.java
deleted file mode 100644
index 5da472e..0000000
--- a/streams-verbs/src/test/java/org/apache/streams/verbs/VerbDefinitionResolverTest.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.verbs;
-
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.pojo.json.Activity;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Test;
-
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Tests for {$link: org.apache.streams.verbs.VerbDefinitionResolver}.
- */
-public class VerbDefinitionResolverTest {
-
- ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- /**
- * Test of matchingVerbDefinitions.
- */
- @Test
- public void testMatchingVerbDefinitions() throws Exception {
- VerbDefinition definition = mapper.readValue(VerbDefinitionResolverTest.class.getResourceAsStream("/post.json"), VerbDefinition.class);
- VerbDefinitionResolver resolver = new VerbDefinitionResolver(Stream.of(definition).collect(Collectors.toSet()));
- Activity activity0 = mapper.readValue("{\"id\":\"1\",\"verb\":\"notpost\"}\n", Activity.class);
- Set<VerbDefinition> result0 = resolver.matchingVerbDefinitions(activity0);
- assert result0.size() == 0;
- Activity activity1 = mapper.readValue("{\"id\":\"1\",\"verb\":\"post\"}\n", Activity.class);
- Set<VerbDefinition> result1 = resolver.matchingVerbDefinitions(activity1);
- assert result1.size() == 1;
- assert result1.contains(definition);
- Activity activity2 = mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"provider\":{\"id\":\"providerId\",\"objectType\":\"product\"}}\n", Activity.class);
- Set<VerbDefinition> result2 = resolver.matchingVerbDefinitions(activity2);
- assert result2.size() == 1;
- assert result2.contains(definition);
- Activity activity3 = mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"provider\":{\"id\":\"providerId\",\"objectType\":\"application\"}}\n\n", Activity.class);
- Set<VerbDefinition> result3 = resolver.matchingVerbDefinitions(activity3);
- assert result3.size() == 1;
- assert result3.contains(definition);
- Activity activity4 = mapper.readValue("{\"id\":\"id\",\"verb\":\"post\",\"object\":{\"id\":\"objectId\",\"objectType\":\"task\"}}\n", Activity.class);
- Set<VerbDefinition> result4 = resolver.matchingVerbDefinitions(activity4);
- assert result4.size() == 1;
- assert result4.contains(definition);
- Activity activity5 = mapper.readValue("{\"id\":\"id\",\"verb\":\"post\",\"target\":{\"id\":\"targetId\",\"objectType\":\"group\"}}\n", Activity.class);
- Set<VerbDefinition> result5 = resolver.matchingVerbDefinitions(activity5);
- assert result5.size() == 1;
- assert result5.contains(definition);
- Activity activity6 = mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"actor\":{\"id\":\"actorId\",\"objectType\":\"page\"}}\n", Activity.class);
- Set<VerbDefinition> result6 = resolver.matchingVerbDefinitions(activity6);
- assert result6.size() == 1;
- assert result6.contains(definition);
- }
-
- /**
- * Test of matchingObjectCombinations.
- */
- @Test
- public void testMatchingObjectCombinations() throws Exception {
- VerbDefinition provider = mapper.readValue(VerbDefinitionResolverTest.class.getResourceAsStream("/provider.json"), VerbDefinition.class);
- VerbDefinition actor = mapper.readValue(VerbDefinitionResolverTest.class.getResourceAsStream("/actor.json"), VerbDefinition.class);
- VerbDefinition object = mapper.readValue(VerbDefinitionResolverTest.class.getResourceAsStream("/object.json"), VerbDefinition.class);
- VerbDefinition post = mapper.readValue(VerbDefinitionResolverTest.class.getResourceAsStream("/post.json"), VerbDefinition.class);
- VerbDefinition follow = mapper.readValue(VerbDefinitionResolverTest.class.getResourceAsStream("/follow.json"), VerbDefinition.class);
- VerbDefinitionResolver resolver = new VerbDefinitionResolver(Stream.of(provider, actor, object, post, follow).collect(Collectors.toSet()));
- Activity activity0 = mapper.readValue("{\"id\":\"1\",\"verb\":\"notpost\"}\n", Activity.class);
- List<ObjectCombination> result0 = resolver.matchingObjectCombinations(activity0);
- assert result0.size() == 0;
- Activity activity1 = mapper.readValue("{\"id\":\"1\",\"verb\":\"post\"}\n", Activity.class);
- List<ObjectCombination> result1 = resolver.matchingObjectCombinations(activity1);
- assert result1.size() == 4;
- Activity activity2 = mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"provider\":{\"id\":\"providerId\",\"objectType\":\"product\"}}\n", Activity.class);
- List<ObjectCombination> result2 = resolver.matchingObjectCombinations(activity2);
- assert result2.size() == 3;
- Activity activity3 = mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"provider\":{\"id\":\"providerId\",\"objectType\":\"application\"}}\n", Activity.class);
- List<ObjectCombination> result3 = resolver.matchingObjectCombinations(activity3);
- assert result3.size() == 4;
- assert provider.getObjects().get(0).equals(result3.get(0));
- Activity activity4 = mapper.readValue("{\"id\":\"id\",\"verb\":\"post\",\"object\":{\"id\":\"objectId\",\"objectType\":\"task\"}}\n", Activity.class);
- List<ObjectCombination> result4 = resolver.matchingObjectCombinations(activity4);
- assert result4.size() == 4;
- assert object.getObjects().get(0).equals(result4.get(0));
- Activity activity5 = mapper.readValue("{\"id\":\"id\",\"verb\":\"post\",\"target\":{\"id\":\"targetId\",\"objectType\":\"group\"}}\n", Activity.class);
- List<ObjectCombination> result5 = resolver.matchingObjectCombinations(activity5);
- assert result5.size() == 4;
- Activity activity6 = mapper.readValue("{\"id\":\"1\",\"verb\":\"post\",\"actor\":{\"id\":\"actorId\",\"objectType\":\"person\"}}\n", Activity.class);
- List<ObjectCombination> result6 = resolver.matchingObjectCombinations(activity6);
- assert result6.size() == 4;
- assert actor.getObjects().get(0).equals(result6.get(0));
- }
-}
\ No newline at end of file
diff --git a/streams-verbs/src/test/java/org/apache/streams/verbs/VerbDefinitionTemplateTest.java b/streams-verbs/src/test/java/org/apache/streams/verbs/VerbDefinitionTemplateTest.java
deleted file mode 100644
index 96039cb..0000000
--- a/streams-verbs/src/test/java/org/apache/streams/verbs/VerbDefinitionTemplateTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.verbs;
-
-import org.apache.streams.pojo.json.Activity;
-import org.apache.streams.pojo.json.ActivityObject;
-import org.apache.streams.pojo.json.Provider;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Test;
-
-/**
- * Tests for {$link: org.apache.streams.verbs.VerbDefinitionTemplateUtil}.
- */
-public class VerbDefinitionTemplateTest {
-
- private ObjectMapper mapper = new ObjectMapper();
-
- /**
- * Test application of template with no field.
- */
- @Test
- public void testNoField() throws Exception {
- Activity activity = new Activity().withVerb("nofields");
- VerbDefinition definition = mapper.readValue(VerbDefinitionResolverTest.class.getResourceAsStream("/nofields.json"), VerbDefinition.class);
- assert VerbDefinitionTemplateUtil.asString(activity, definition.getObjects().get(0)).contains("something");
- }
-
- /**
- * Test application of template with top-level fields.
- */
- @Test
- public void testTopField() throws Exception {
- ActivityObject actor = new ActivityObject();
- actor.setObjectType("page");
- actor.setDisplayName("Paige");
- Provider provider = new Provider();
- provider.setObjectType("application");
- provider.setDisplayName("Ahp");
- ActivityObject object = new ActivityObject();
- object.setObjectType("task");
- object.setDisplayName("Tsk");
- ActivityObject target = new ActivityObject();
- target.setObjectType("person");
- target.setDisplayName("Homie");
- Activity activity = new Activity().withVerb("post");
- activity.setActor(actor);
- activity.setProvider(provider);
- activity.setObject(object);
- activity.setTarget(target);
- VerbDefinition definition = mapper.readValue(VerbDefinitionTest.class.getResourceAsStream("/post.json"), VerbDefinition.class);
- String message = VerbDefinitionTemplateUtil.asString(activity, definition.getObjects().get(0));
- assert message.contains("Paige");
- assert message.contains("Ahp");
- assert message.contains("Tsk");
- assert message.contains("Homie");
- }
-
- /**
- * Test application of template with second-level fields.
- */
- @Test
- public void testSecondFields() {
-
- }
-
-}
diff --git a/streams-verbs/src/test/java/org/apache/streams/verbs/VerbDefinitionTest.java b/streams-verbs/src/test/java/org/apache/streams/verbs/VerbDefinitionTest.java
deleted file mode 100644
index 300115b..0000000
--- a/streams-verbs/src/test/java/org/apache/streams/verbs/VerbDefinitionTest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.streams.verbs;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.junit.Test;
-
-/**
- * Unit tests for VerbDefinition and utils.
- */
-public class VerbDefinitionTest {
-
- private ObjectMapper mapper = new ObjectMapper();
-
- /**
- * Test read verb definition from json.
- */
- @Test
- public void testReadVerbDefinitionJson() throws Exception {
-
- VerbDefinition definition = mapper.readValue(VerbDefinitionTest.class.getResourceAsStream("/do.json"), VerbDefinition.class);
-
- assert definition != null;
- assert definition.getObjectType().equals("verb");
- assert definition.getObjects().size() == 1;
- assert definition.getObjects().get(0).getActor().equals("*");
- assert definition.getObjects().get(0).getObject().equals("*");
- assert definition.getObjects().get(0).getTarget().equals("*");
- assert definition.getObjects().get(0).getProvider().equals("*");
- assert definition.getObjects().get(0).getTemplates().getAdditionalProperties().size() == 1;
- }
-
- /**
- * Test verb definition defaults are set.
- */
- @Test
- public void testObjectCombinationDefaults() throws Exception {
-
- ObjectCombination combination = new ObjectCombination();
-
- assert combination.getActor().equals("*");
- assert combination.getObject().equals("*");
- assert combination.getTarget().equals("*");
- assert combination.getProvider().equals("*");
- assert !combination.getTargetRequired();
-
- }
-
-}
diff --git a/streams-verbs/src/test/resources/activities.txt b/streams-verbs/src/test/resources/activities.txt
deleted file mode 100644
index ecab0e5..0000000
--- a/streams-verbs/src/test/resources/activities.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-{"id":"1","verb":"do","provider":{"id":"providerId","objectType":"product"}}
-{"id":"2","verb":"do","actor":{"id":"actorId","objectType":"person"}}
-{"id":"3","verb":"do","object":{"id":"objectId","objectType":"task"}}
-{"id":"4","verb":"do","target":{"id":"targetId","objectType":"group"}}
diff --git a/streams-verbs/src/test/resources/actor.json b/streams-verbs/src/test/resources/actor.json
deleted file mode 100644
index f7b62a8..0000000
--- a/streams-verbs/src/test/resources/actor.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/targetrequired",
- "value": "post",
- "objects": [
- {
- "actor": "person"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-verbs/src/test/resources/do.json b/streams-verbs/src/test/resources/do.json
deleted file mode 100644
index f959a1c..0000000
--- a/streams-verbs/src/test/resources/do.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/do",
- "value": "do",
- "objects": [
- {
- "actor": "*",
- "object": "*",
- "target": "*",
- "templates": {
- "*": "{actor} do {object} to {target}"
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-verbs/src/test/resources/follow.json b/streams-verbs/src/test/resources/follow.json
deleted file mode 100644
index 082c65f..0000000
--- a/streams-verbs/src/test/resources/follow.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/targetrequired",
- "value": "follow",
- "objects": [
- {
- "actorRequired": "true",
- "actor": "page",
- "object": "page",
- "objectRequired": "true"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-verbs/src/test/resources/nofields.json b/streams-verbs/src/test/resources/nofields.json
deleted file mode 100644
index fd69a49..0000000
--- a/streams-verbs/src/test/resources/nofields.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/post",
- "value": "nofields",
- "objects": [
- {
- "actor": "*",
- "object": "*",
- "provider": "*",
- "target": "*",
- "templates": {
- "*": "something did something to something"
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-verbs/src/test/resources/object.json b/streams-verbs/src/test/resources/object.json
deleted file mode 100644
index 36eed78..0000000
--- a/streams-verbs/src/test/resources/object.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/targetrequired",
- "value": "post",
- "objects": [
- {
- "object": "task"
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-verbs/src/test/resources/post.json b/streams-verbs/src/test/resources/post.json
deleted file mode 100644
index 902e6c8..0000000
--- a/streams-verbs/src/test/resources/post.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/post",
- "value": "post",
- "objects": [
- {
- "actor": "*",
- "object": "*",
- "provider": "*",
- "target": "*",
- "templates": {
- "*": "[{provider}] {actor} post {object} to {target}"
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/streams-verbs/src/test/resources/provider.json b/streams-verbs/src/test/resources/provider.json
deleted file mode 100644
index c5ec184..0000000
--- a/streams-verbs/src/test/resources/provider.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "id": "http://streams.apache.org/verbs/test/targetrequired",
- "value": "post",
- "objects": [
- {
- "provider": "application"
- }
- ]
-}
\ No newline at end of file
--
To stop receiving notification emails like this one, please contact
['"commits@streams.apache.org" <co...@streams.apache.org>'].