You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2017/04/05 23:19:24 UTC
[21/23] storm git commit: STORM-2453 Move non-connectors into the top
directory
STORM-2453 Move non-connectors into the top directory
* move storm-sql, flux, storm-submit-tools to the top directory
* rearrange target directories for them in binary dist.
* create "lib-tools" directory for storing non-storm-core libs
* sql: lib-tools/sql/core and lib-tools/sql/runtime
* storm-submit-tools: lib-tools/submit-tools
* storm-kafka-monitor: lib-tools/storm-kafka-monitor
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e9d78338
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e9d78338
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e9d78338
Branch: refs/heads/master
Commit: e9d783386e6d964df925546beb4db55106e61312
Parents: 28c8711
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Apr 4 17:27:32 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Apr 4 23:04:18 2017 +0900
----------------------------------------------------------------------
bin/storm-kafka-monitor | 2 +-
bin/storm.py | 9 +-
external/flux/README.md | 872 -------------------
external/flux/flux-core/pom.xml | 112 ---
.../main/java/org/apache/storm/flux/Flux.java | 261 ------
.../java/org/apache/storm/flux/FluxBuilder.java | 630 --------------
.../apache/storm/flux/api/TopologySource.java | 39 -
.../org/apache/storm/flux/model/BeanDef.java | 39 -
.../storm/flux/model/BeanListReference.java | 37 -
.../apache/storm/flux/model/BeanReference.java | 39 -
.../org/apache/storm/flux/model/BoltDef.java | 24 -
.../storm/flux/model/ConfigMethodDef.java | 67 --
.../storm/flux/model/ExecutionContext.java | 77 --
.../apache/storm/flux/model/GroupingDef.java | 77 --
.../org/apache/storm/flux/model/IncludeDef.java | 54 --
.../org/apache/storm/flux/model/ObjectDef.java | 95 --
.../apache/storm/flux/model/PropertyDef.java | 58 --
.../org/apache/storm/flux/model/SpoutDef.java | 24 -
.../org/apache/storm/flux/model/StreamDef.java | 64 --
.../apache/storm/flux/model/TopologyDef.java | 216 -----
.../storm/flux/model/TopologySourceDef.java | 36 -
.../org/apache/storm/flux/model/VertexDef.java | 36 -
.../apache/storm/flux/parser/FluxParser.java | 208 -----
.../flux-core/src/main/resources/splash.txt | 9 -
.../org/apache/storm/flux/FluxBuilderTest.java | 31 -
.../org/apache/storm/flux/IntegrationTest.java | 39 -
.../java/org/apache/storm/flux/TCKTest.java | 256 ------
.../multilang/MultilangEnvirontmentTest.java | 89 --
.../apache/storm/flux/test/SimpleTopology.java | 59 --
.../storm/flux/test/SimpleTopologySource.java | 52 --
.../test/SimpleTopologyWithConfigParam.java | 55 --
.../org/apache/storm/flux/test/TestBolt.java | 131 ---
.../storm/flux/test/TridentTopologySource.java | 71 --
.../src/test/resources/configs/bad_hbase.yaml | 98 ---
.../test/resources/configs/bad_shell_test.yaml | 118 ---
.../resources/configs/config-methods-test.yaml | 92 --
.../resources/configs/diamond-topology.yaml | 87 --
.../existing-topology-method-override.yaml | 25 -
.../existing-topology-reflection-config.yaml | 24 -
.../configs/existing-topology-reflection.yaml | 24 -
.../configs/existing-topology-trident.yaml | 24 -
.../resources/configs/existing-topology.yaml | 23 -
.../src/test/resources/configs/hdfs_test.yaml | 97 ---
.../test/resources/configs/include_test.yaml | 25 -
.../configs/invalid-existing-topology.yaml | 33 -
.../src/test/resources/configs/kafka_test.yaml | 126 ---
.../src/test/resources/configs/shell_test.yaml | 118 ---
.../test/resources/configs/simple_hbase.yaml | 120 ---
.../resources/configs/substitution-test.yaml | 106 ---
.../src/test/resources/configs/tck.yaml | 95 --
.../src/test/resources/configs/test.properties | 18 -
.../flux-core/src/test/resources/log4j2.xml | 34 -
external/flux/flux-examples/README.md | 93 --
external/flux/flux-examples/pom.xml | 146 ----
.../flux/examples/StatefulWordCounter.java | 64 --
.../storm/flux/examples/TestPrintBolt.java | 39 -
.../storm/flux/examples/TestWindowBolt.java | 47 -
.../storm/flux/examples/WordCountClient.java | 74 --
.../apache/storm/flux/examples/WordCounter.java | 71 --
.../src/main/resources/hbase_bolt.properties | 18 -
.../src/main/resources/hdfs_bolt.properties | 26 -
.../src/main/resources/kafka_spout.yaml | 136 ---
.../src/main/resources/multilang.yaml | 89 --
.../src/main/resources/simple_hbase.yaml | 92 --
.../src/main/resources/simple_hdfs.yaml | 105 ---
.../resources/simple_stateful_wordcount.yaml | 60 --
.../src/main/resources/simple_windowing.yaml | 69 --
.../src/main/resources/simple_wordcount.yaml | 68 --
external/flux/flux-ui/README.md | 3 -
external/flux/flux-wrappers/pom.xml | 51 --
.../flux/wrappers/bolts/FluxShellBolt.java | 175 ----
.../storm/flux/wrappers/bolts/LogInfoBolt.java | 44 -
.../flux/wrappers/spouts/FluxShellSpout.java | 175 ----
.../main/resources/resources/randomsentence.js | 93 --
.../main/resources/resources/splitsentence.py | 24 -
external/flux/pom.xml | 85 --
external/sql/README.md | 207 -----
external/sql/pom.xml | 47 -
external/sql/storm-sql-core/pom.xml | 279 ------
.../sql/storm-sql-core/src/codegen/config.fmpp | 23 -
.../storm-sql-core/src/codegen/data/Parser.tdd | 80 --
.../src/codegen/includes/license.ftl | 17 -
.../src/codegen/includes/parserImpls.ftl | 113 ---
.../storm/sql/AbstractTridentProcessor.java | 58 --
.../src/jvm/org/apache/storm/sql/StormSql.java | 59 --
.../jvm/org/apache/storm/sql/StormSqlImpl.java | 290 ------
.../org/apache/storm/sql/StormSqlRunner.java | 85 --
.../sql/calcite/ParallelStreamableTable.java | 35 -
.../apache/storm/sql/compiler/CompilerUtil.java | 183 ----
.../sql/compiler/RexNodeToJavaCodeCompiler.java | 231 -----
.../sql/compiler/StormSqlTypeFactoryImpl.java | 51 --
.../standalone/BuiltinAggregateFunctions.java | 238 -----
.../backends/standalone/PlanCompiler.java | 139 ---
.../standalone/PostOrderRelNodeVisitor.java | 132 ---
.../backends/standalone/RelNodeCompiler.java | 484 ----------
.../storm/sql/javac/CompilingClassLoader.java | 225 -----
.../storm/sql/parser/ColumnConstraint.java | 42 -
.../storm/sql/parser/ColumnDefinition.java | 44 -
.../storm/sql/parser/SqlCreateFunction.java | 94 --
.../apache/storm/sql/parser/SqlCreateTable.java | 166 ----
.../apache/storm/sql/parser/SqlDDLKeywords.java | 27 -
.../apache/storm/sql/parser/StormParser.java | 47 -
.../apache/storm/sql/parser/UnparseUtil.java | 60 --
.../sql/planner/StormRelDataTypeSystem.java | 37 -
.../apache/storm/sql/planner/StormRelUtils.java | 68 --
.../planner/UnsupportedOperatorsVisitor.java | 24 -
.../storm/sql/planner/rel/StormCalcRelBase.java | 32 -
.../sql/planner/rel/StormFilterRelBase.java | 32 -
.../storm/sql/planner/rel/StormJoinRelBase.java | 36 -
.../sql/planner/rel/StormProjectRelBase.java | 35 -
.../storm/sql/planner/rel/StormRelNode.java | 24 -
.../planner/rel/StormStreamInsertRelBase.java | 36 -
.../sql/planner/rel/StormStreamScanRelBase.java | 34 -
.../storm/sql/planner/trident/QueryPlanner.java | 156 ----
.../sql/planner/trident/TridentPlanCreator.java | 125 ---
.../planner/trident/TridentStormRuleSets.java | 110 ---
.../sql/planner/trident/rel/TridentCalcRel.java | 97 ---
.../planner/trident/rel/TridentFilterRel.java | 65 --
.../trident/rel/TridentLogicalConvention.java | 67 --
.../planner/trident/rel/TridentProjectRel.java | 69 --
.../sql/planner/trident/rel/TridentRel.java | 26 -
.../trident/rel/TridentStreamInsertRel.java | 76 --
.../trident/rel/TridentStreamScanRel.java | 55 --
.../trident/rules/TridentAggregateRule.java | 39 -
.../planner/trident/rules/TridentCalcRule.java | 45 -
.../trident/rules/TridentFilterRule.java | 45 -
.../planner/trident/rules/TridentJoinRule.java | 37 -
.../trident/rules/TridentModifyRule.java | 71 --
.../trident/rules/TridentProjectRule.java | 45 -
.../planner/trident/rules/TridentScanRule.java | 58 --
.../test/org/apache/storm/sql/TestStormSql.java | 492 -----------
.../storm/sql/compiler/TestExprSemantic.java | 410 ---------
.../backends/standalone/TestCompilerUtils.java | 183 ----
.../backends/standalone/TestPlanCompiler.java | 104 ---
.../standalone/TestRelNodeCompiler.java | 64 --
.../backends/trident/TestCompilerUtils.java | 208 -----
.../backends/trident/TestPlanCompiler.java | 235 -----
.../apache/storm/sql/parser/TestSqlParser.java | 54 --
.../storm-sql-external/storm-sql-hdfs/pom.xml | 104 ---
.../storm/sql/hdfs/HdfsDataSourcesProvider.java | 135 ---
...apache.storm.sql.runtime.DataSourcesProvider | 16 -
.../sql/hdfs/TestHdfsDataSourcesProvider.java | 129 ---
.../storm-sql-external/storm-sql-kafka/pom.xml | 93 --
.../sql/kafka/KafkaDataSourcesProvider.java | 170 ----
...apache.storm.sql.runtime.DataSourcesProvider | 16 -
.../sql/kafka/TestKafkaDataSourcesProvider.java | 125 ---
.../storm-sql-mongodb/pom.xml | 84 --
.../sql/mongodb/MongoDataSourcesProvider.java | 126 ---
...apache.storm.sql.runtime.DataSourcesProvider | 16 -
.../mongodb/TestMongoDataSourcesProvider.java | 122 ---
.../storm-sql-external/storm-sql-redis/pom.xml | 76 --
.../sql/redis/RedisDataSourcesProvider.java | 228 -----
...apache.storm.sql.runtime.DataSourcesProvider | 16 -
.../sql/redis/TestRedisDataSourcesProvider.java | 158 ----
external/sql/storm-sql-runtime/pom.xml | 136 ---
.../calcite/interpreter/StormContext.java | 31 -
.../sql/runtime/AbstractChannelHandler.java | 52 --
.../sql/runtime/AbstractValuesProcessor.java | 46 -
.../storm/sql/runtime/ChannelContext.java | 30 -
.../storm/sql/runtime/ChannelHandler.java | 41 -
.../org/apache/storm/sql/runtime/Channels.java | 109 ---
.../apache/storm/sql/runtime/DataSource.java | 27 -
.../storm/sql/runtime/DataSourcesProvider.java | 47 -
.../storm/sql/runtime/DataSourcesRegistry.java | 76 --
.../org/apache/storm/sql/runtime/FieldInfo.java | 47 -
.../storm/sql/runtime/IOutputSerializer.java | 31 -
.../sql/runtime/ISqlTridentDataSource.java | 65 --
.../sql/runtime/SimpleSqlTridentConsumer.java | 41 -
.../storm/sql/runtime/StormSqlFunctions.java | 34 -
.../calcite/DebuggableExecutableExpression.java | 45 -
.../runtime/calcite/ExecutableExpression.java | 31 -
.../sql/runtime/calcite/StormDataContext.java | 79 --
.../socket/SocketDataSourcesProvider.java | 101 ---
.../datasource/socket/trident/SocketState.java | 89 --
.../socket/trident/SocketStateUpdater.java | 59 --
.../socket/trident/TridentSocketSpout.java | 177 ----
.../sql/runtime/serde/avro/AvroScheme.java | 74 --
.../sql/runtime/serde/avro/AvroSerializer.java | 72 --
.../sql/runtime/serde/avro/CachedSchemas.java | 41 -
.../storm/sql/runtime/serde/csv/CsvScheme.java | 70 --
.../sql/runtime/serde/csv/CsvSerializer.java | 59 --
.../sql/runtime/serde/json/JsonScheme.java | 58 --
.../sql/runtime/serde/json/JsonSerializer.java | 57 --
.../storm/sql/runtime/serde/tsv/TsvScheme.java | 58 --
.../sql/runtime/serde/tsv/TsvSerializer.java | 54 --
.../trident/functions/EvaluationCalc.java | 86 --
.../trident/functions/EvaluationFilter.java | 61 --
.../trident/functions/EvaluationFunction.java | 67 --
.../trident/functions/ForwardFunction.java | 30 -
.../storm/sql/runtime/utils/FieldInfoUtils.java | 39 -
.../storm/sql/runtime/utils/SerdeUtils.java | 123 ---
.../apache/storm/sql/runtime/utils/Utils.java | 55 --
...apache.storm.sql.runtime.DataSourcesProvider | 16 -
.../apache/storm/sql/TestAvroSerializer.java | 72 --
.../org/apache/storm/sql/TestCsvSerializer.java | 54 --
.../storm/sql/TestJsonRepresentation.java | 52 --
.../org/apache/storm/sql/TestTsvSerializer.java | 46 -
.../test/org/apache/storm/sql/TestUtils.java | 584 -------------
.../socket/TestSocketDataSourceProvider.java | 94 --
external/storm-submit-tools/pom.xml | 216 -----
.../submit/command/DependencyResolverMain.java | 158 ----
.../storm/submit/dependency/AetherUtils.java | 91 --
.../apache/storm/submit/dependency/Booter.java | 51 --
.../submit/dependency/DependencyResolver.java | 98 ---
.../dependency/RepositorySystemFactory.java | 67 --
.../submit/dependency/AetherUtilsTest.java | 102 ---
.../dependency/DependencyResolverTest.java | 83 --
flux/README.md | 872 +++++++++++++++++++
flux/flux-core/pom.xml | 112 +++
.../main/java/org/apache/storm/flux/Flux.java | 259 ++++++
.../java/org/apache/storm/flux/FluxBuilder.java | 630 ++++++++++++++
.../apache/storm/flux/api/TopologySource.java | 39 +
.../org/apache/storm/flux/model/BeanDef.java | 34 +
.../storm/flux/model/BeanListReference.java | 37 +
.../apache/storm/flux/model/BeanReference.java | 39 +
.../org/apache/storm/flux/model/BoltDef.java | 24 +
.../storm/flux/model/ConfigMethodDef.java | 67 ++
.../storm/flux/model/ExecutionContext.java | 77 ++
.../apache/storm/flux/model/GroupingDef.java | 77 ++
.../org/apache/storm/flux/model/IncludeDef.java | 54 ++
.../org/apache/storm/flux/model/ObjectDef.java | 95 ++
.../apache/storm/flux/model/PropertyDef.java | 58 ++
.../org/apache/storm/flux/model/SpoutDef.java | 24 +
.../org/apache/storm/flux/model/StreamDef.java | 64 ++
.../apache/storm/flux/model/TopologyDef.java | 216 +++++
.../storm/flux/model/TopologySourceDef.java | 36 +
.../org/apache/storm/flux/model/VertexDef.java | 36 +
.../apache/storm/flux/parser/FluxParser.java | 206 +++++
flux/flux-core/src/main/resources/splash.txt | 9 +
.../org/apache/storm/flux/FluxBuilderTest.java | 31 +
.../org/apache/storm/flux/IntegrationTest.java | 39 +
.../java/org/apache/storm/flux/TCKTest.java | 254 ++++++
.../multilang/MultilangEnvirontmentTest.java | 89 ++
.../apache/storm/flux/test/SimpleTopology.java | 58 ++
.../storm/flux/test/SimpleTopologySource.java | 52 ++
.../test/SimpleTopologyWithConfigParam.java | 53 ++
.../org/apache/storm/flux/test/TestBolt.java | 131 +++
.../storm/flux/test/TridentTopologySource.java | 71 ++
.../src/test/resources/configs/bad_hbase.yaml | 98 +++
.../test/resources/configs/bad_shell_test.yaml | 118 +++
.../resources/configs/config-methods-test.yaml | 92 ++
.../resources/configs/diamond-topology.yaml | 87 ++
.../existing-topology-method-override.yaml | 25 +
.../existing-topology-reflection-config.yaml | 24 +
.../configs/existing-topology-reflection.yaml | 24 +
.../configs/existing-topology-trident.yaml | 24 +
.../resources/configs/existing-topology.yaml | 23 +
.../src/test/resources/configs/hdfs_test.yaml | 97 +++
.../test/resources/configs/include_test.yaml | 25 +
.../configs/invalid-existing-topology.yaml | 33 +
.../src/test/resources/configs/kafka_test.yaml | 126 +++
.../src/test/resources/configs/shell_test.yaml | 118 +++
.../test/resources/configs/simple_hbase.yaml | 120 +++
.../resources/configs/substitution-test.yaml | 106 +++
.../src/test/resources/configs/tck.yaml | 95 ++
.../src/test/resources/configs/test.properties | 18 +
flux/flux-core/src/test/resources/log4j2.xml | 34 +
flux/flux-examples/README.md | 93 ++
flux/flux-examples/pom.xml | 146 ++++
.../flux/examples/StatefulWordCounter.java | 64 ++
.../storm/flux/examples/TestPrintBolt.java | 39 +
.../storm/flux/examples/TestWindowBolt.java | 47 +
.../storm/flux/examples/WordCountClient.java | 74 ++
.../apache/storm/flux/examples/WordCounter.java | 71 ++
.../src/main/resources/hbase_bolt.properties | 18 +
.../src/main/resources/hdfs_bolt.properties | 26 +
.../src/main/resources/kafka_spout.yaml | 136 +++
.../src/main/resources/multilang.yaml | 89 ++
.../src/main/resources/simple_hbase.yaml | 92 ++
.../src/main/resources/simple_hdfs.yaml | 105 +++
.../resources/simple_stateful_wordcount.yaml | 60 ++
.../src/main/resources/simple_windowing.yaml | 69 ++
.../src/main/resources/simple_wordcount.yaml | 68 ++
flux/flux-ui/README.md | 3 +
flux/flux-wrappers/pom.xml | 51 ++
.../flux/wrappers/bolts/FluxShellBolt.java | 175 ++++
.../storm/flux/wrappers/bolts/LogInfoBolt.java | 44 +
.../flux/wrappers/spouts/FluxShellSpout.java | 175 ++++
.../main/resources/resources/randomsentence.js | 93 ++
.../main/resources/resources/splitsentence.py | 24 +
flux/pom.xml | 85 ++
pom.xml | 6 +-
sql/README.md | 207 +++++
sql/pom.xml | 47 +
sql/storm-sql-core/pom.xml | 279 ++++++
sql/storm-sql-core/src/codegen/config.fmpp | 23 +
sql/storm-sql-core/src/codegen/data/Parser.tdd | 80 ++
.../src/codegen/includes/license.ftl | 17 +
.../src/codegen/includes/parserImpls.ftl | 113 +++
.../storm/sql/AbstractTridentProcessor.java | 56 ++
.../src/jvm/org/apache/storm/sql/StormSql.java | 59 ++
.../jvm/org/apache/storm/sql/StormSqlImpl.java | 290 ++++++
.../org/apache/storm/sql/StormSqlRunner.java | 85 ++
.../sql/calcite/ParallelStreamableTable.java | 35 +
.../apache/storm/sql/compiler/CompilerUtil.java | 183 ++++
.../sql/compiler/RexNodeToJavaCodeCompiler.java | 231 +++++
.../sql/compiler/StormSqlTypeFactoryImpl.java | 51 ++
.../standalone/BuiltinAggregateFunctions.java | 238 +++++
.../backends/standalone/PlanCompiler.java | 139 +++
.../standalone/PostOrderRelNodeVisitor.java | 132 +++
.../backends/standalone/RelNodeCompiler.java | 484 ++++++++++
.../storm/sql/javac/CompilingClassLoader.java | 225 +++++
.../storm/sql/parser/ColumnConstraint.java | 42 +
.../storm/sql/parser/ColumnDefinition.java | 44 +
.../storm/sql/parser/SqlCreateFunction.java | 94 ++
.../apache/storm/sql/parser/SqlCreateTable.java | 166 ++++
.../apache/storm/sql/parser/SqlDDLKeywords.java | 27 +
.../apache/storm/sql/parser/StormParser.java | 47 +
.../apache/storm/sql/parser/UnparseUtil.java | 59 ++
.../sql/planner/StormRelDataTypeSystem.java | 37 +
.../apache/storm/sql/planner/StormRelUtils.java | 68 ++
.../planner/UnsupportedOperatorsVisitor.java | 24 +
.../storm/sql/planner/rel/StormCalcRelBase.java | 31 +
.../sql/planner/rel/StormFilterRelBase.java | 31 +
.../storm/sql/planner/rel/StormJoinRelBase.java | 35 +
.../sql/planner/rel/StormProjectRelBase.java | 34 +
.../storm/sql/planner/rel/StormRelNode.java | 24 +
.../planner/rel/StormStreamInsertRelBase.java | 35 +
.../sql/planner/rel/StormStreamScanRelBase.java | 33 +
.../storm/sql/planner/trident/QueryPlanner.java | 156 ++++
.../sql/planner/trident/TridentPlanCreator.java | 123 +++
.../planner/trident/TridentStormRuleSets.java | 110 +++
.../sql/planner/trident/rel/TridentCalcRel.java | 97 +++
.../planner/trident/rel/TridentFilterRel.java | 65 ++
.../trident/rel/TridentLogicalConvention.java | 67 ++
.../planner/trident/rel/TridentProjectRel.java | 69 ++
.../sql/planner/trident/rel/TridentRel.java | 25 +
.../trident/rel/TridentStreamInsertRel.java | 76 ++
.../trident/rel/TridentStreamScanRel.java | 55 ++
.../trident/rules/TridentAggregateRule.java | 39 +
.../planner/trident/rules/TridentCalcRule.java | 45 +
.../trident/rules/TridentFilterRule.java | 45 +
.../planner/trident/rules/TridentJoinRule.java | 37 +
.../trident/rules/TridentModifyRule.java | 71 ++
.../trident/rules/TridentProjectRule.java | 45 +
.../planner/trident/rules/TridentScanRule.java | 58 ++
.../test/org/apache/storm/sql/TestStormSql.java | 492 +++++++++++
.../storm/sql/compiler/TestExprSemantic.java | 410 +++++++++
.../backends/standalone/TestCompilerUtils.java | 183 ++++
.../backends/standalone/TestPlanCompiler.java | 104 +++
.../standalone/TestRelNodeCompiler.java | 64 ++
.../backends/trident/TestCompilerUtils.java | 208 +++++
.../backends/trident/TestPlanCompiler.java | 232 +++++
.../apache/storm/sql/parser/TestSqlParser.java | 54 ++
sql/storm-sql-external/storm-sql-hdfs/pom.xml | 104 +++
.../storm/sql/hdfs/HdfsDataSourcesProvider.java | 135 +++
...apache.storm.sql.runtime.DataSourcesProvider | 16 +
.../sql/hdfs/TestHdfsDataSourcesProvider.java | 129 +++
sql/storm-sql-external/storm-sql-kafka/pom.xml | 93 ++
.../sql/kafka/KafkaDataSourcesProvider.java | 170 ++++
...apache.storm.sql.runtime.DataSourcesProvider | 16 +
.../sql/kafka/TestKafkaDataSourcesProvider.java | 125 +++
.../storm-sql-mongodb/pom.xml | 84 ++
.../sql/mongodb/MongoDataSourcesProvider.java | 126 +++
...apache.storm.sql.runtime.DataSourcesProvider | 16 +
.../mongodb/TestMongoDataSourcesProvider.java | 122 +++
sql/storm-sql-external/storm-sql-redis/pom.xml | 76 ++
.../sql/redis/RedisDataSourcesProvider.java | 228 +++++
...apache.storm.sql.runtime.DataSourcesProvider | 16 +
.../sql/redis/TestRedisDataSourcesProvider.java | 158 ++++
sql/storm-sql-runtime/pom.xml | 136 +++
.../calcite/interpreter/StormContext.java | 31 +
.../sql/runtime/AbstractChannelHandler.java | 52 ++
.../sql/runtime/AbstractValuesProcessor.java | 44 +
.../storm/sql/runtime/ChannelContext.java | 30 +
.../storm/sql/runtime/ChannelHandler.java | 41 +
.../org/apache/storm/sql/runtime/Channels.java | 109 +++
.../apache/storm/sql/runtime/DataSource.java | 27 +
.../storm/sql/runtime/DataSourcesProvider.java | 47 +
.../storm/sql/runtime/DataSourcesRegistry.java | 76 ++
.../org/apache/storm/sql/runtime/FieldInfo.java | 47 +
.../storm/sql/runtime/IOutputSerializer.java | 31 +
.../sql/runtime/ISqlTridentDataSource.java | 65 ++
.../sql/runtime/SimpleSqlTridentConsumer.java | 41 +
.../storm/sql/runtime/StormSqlFunctions.java | 34 +
.../calcite/DebuggableExecutableExpression.java | 45 +
.../runtime/calcite/ExecutableExpression.java | 31 +
.../sql/runtime/calcite/StormDataContext.java | 79 ++
.../socket/SocketDataSourcesProvider.java | 100 +++
.../datasource/socket/trident/SocketState.java | 89 ++
.../socket/trident/SocketStateUpdater.java | 59 ++
.../socket/trident/TridentSocketSpout.java | 177 ++++
.../sql/runtime/serde/avro/AvroScheme.java | 74 ++
.../sql/runtime/serde/avro/AvroSerializer.java | 72 ++
.../sql/runtime/serde/avro/CachedSchemas.java | 41 +
.../storm/sql/runtime/serde/csv/CsvScheme.java | 70 ++
.../sql/runtime/serde/csv/CsvSerializer.java | 59 ++
.../sql/runtime/serde/json/JsonScheme.java | 58 ++
.../sql/runtime/serde/json/JsonSerializer.java | 57 ++
.../storm/sql/runtime/serde/tsv/TsvScheme.java | 58 ++
.../sql/runtime/serde/tsv/TsvSerializer.java | 54 ++
.../trident/functions/EvaluationCalc.java | 86 ++
.../trident/functions/EvaluationFilter.java | 61 ++
.../trident/functions/EvaluationFunction.java | 67 ++
.../trident/functions/ForwardFunction.java | 30 +
.../storm/sql/runtime/utils/FieldInfoUtils.java | 39 +
.../storm/sql/runtime/utils/SerdeUtils.java | 123 +++
.../apache/storm/sql/runtime/utils/Utils.java | 55 ++
...apache.storm.sql.runtime.DataSourcesProvider | 16 +
.../apache/storm/sql/TestAvroSerializer.java | 72 ++
.../org/apache/storm/sql/TestCsvSerializer.java | 54 ++
.../storm/sql/TestJsonRepresentation.java | 52 ++
.../org/apache/storm/sql/TestTsvSerializer.java | 46 +
.../test/org/apache/storm/sql/TestUtils.java | 584 +++++++++++++
.../socket/TestSocketDataSourceProvider.java | 94 ++
.../final-package/src/main/assembly/binary.xml | 86 +-
storm-submit-tools/pom.xml | 216 +++++
.../submit/command/DependencyResolverMain.java | 158 ++++
.../storm/submit/dependency/AetherUtils.java | 91 ++
.../apache/storm/submit/dependency/Booter.java | 51 ++
.../submit/dependency/DependencyResolver.java | 98 +++
.../dependency/RepositorySystemFactory.java | 67 ++
.../submit/dependency/AetherUtilsTest.java | 102 +++
.../dependency/DependencyResolverTest.java | 83 ++
414 files changed, 19540 insertions(+), 19605 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e9d78338/bin/storm-kafka-monitor
----------------------------------------------------------------------
diff --git a/bin/storm-kafka-monitor b/bin/storm-kafka-monitor
index a51052d..2958e85 100755
--- a/bin/storm-kafka-monitor
+++ b/bin/storm-kafka-monitor
@@ -49,4 +49,4 @@ if [ -z "$JAVA_HOME" ]; then
else
JAVA="$JAVA_HOME/bin/java"
fi
-exec $JAVA $STORM_JAAS_CONF_PARAM -cp $STORM_BASE_DIR/toollib/storm-kafka-monitor-*.jar org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@"
\ No newline at end of file
+exec $JAVA $STORM_JAAS_CONF_PARAM -cp $STORM_BASE_DIR/lib-tools/storm-kafka-monitor/storm-kafka-monitor-*.jar org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/e9d78338/bin/storm.py
----------------------------------------------------------------------
diff --git a/bin/storm.py b/bin/storm.py
index 7ab1f9c..a2d9243 100755
--- a/bin/storm.py
+++ b/bin/storm.py
@@ -82,6 +82,7 @@ if (not os.path.isfile(os.path.join(USER_CONF_DIR, "storm.yaml"))):
STORM_LIB_DIR = os.path.join(STORM_DIR, "lib")
STORM_DRPC_LIB_DIR = os.path.join(STORM_DIR, "lib-drpc-server")
+STORM_TOOLS_LIB_DIR = os.path.join(STORM_DIR, "lib-tools")
STORM_BIN_DIR = os.path.join(STORM_DIR, "bin")
STORM_LOG4J2_CONF_DIR = os.path.join(STORM_DIR, "log4j2")
STORM_SUPERVISOR_LOG_FILE = os.getenv('STORM_SUPERVISOR_LOG_FILE', "supervisor.log")
@@ -167,10 +168,8 @@ def resolve_dependencies(artifacts, artifact_repositories):
print("Resolving dependencies on demand: artifacts (%s) with repositories (%s)" % (artifacts, artifact_repositories))
sys.stdout.flush()
- # TODO: should we move some external modules to outer place?
-
# storm-submit module doesn't rely on storm-core and relevant libs
- extrajars = get_jars_full(STORM_DIR + "/external/storm-submit-tools")
+ extrajars = get_jars_full(os.path.join(STORM_TOOLS_LIB_DIR, "submit-tools"))
classpath = normclasspath(os.pathsep.join(extrajars))
command = [
@@ -343,8 +342,8 @@ def sql(sql_file, topology_name):
local_jars = DEP_JARS_OPTS
artifact_to_file_jars = resolve_dependencies(DEP_ARTIFACTS_OPTS, DEP_ARTIFACTS_REPOSITORIES_OPTS)
- sql_core_jars = get_jars_full(STORM_DIR + "/external/sql/storm-sql-core")
- sql_runtime_jars = get_jars_full(STORM_DIR + "/external/sql/storm-sql-runtime")
+ sql_core_jars = get_jars_full(os.path.join(STORM_TOOLS_LIB_DIR, "sql", "core"))
+ sql_runtime_jars = get_jars_full(os.path.join(STORM_TOOLS_LIB_DIR, "sql", "runtime"))
# include storm-sql-runtime jar(s) to local jar list
local_jars.extend(sql_runtime_jars)
http://git-wip-us.apache.org/repos/asf/storm/blob/e9d78338/external/flux/README.md
----------------------------------------------------------------------
diff --git a/external/flux/README.md b/external/flux/README.md
deleted file mode 100644
index 206ae22..0000000
--- a/external/flux/README.md
+++ /dev/null
@@ -1,872 +0,0 @@
-# flux
-A framework for creating and deploying Apache Storm streaming computations with less friction.
-
-## Definition
-**flux** |fl\u0259ks| _noun_
-
-1. The action or process of flowing or flowing out
-2. Continuous change
-3. In physics, the rate of flow of a fluid, radiant energy, or particles across a given area
-4. A substance mixed with a solid to lower its melting point
-
-## Rationale
-Bad things happen when configuration is hard-coded. No one should have to recompile or repackage an application in
-order to change configuration.
-
-## About
-Flux is a framework and set of utilities that make defining and deploying Apache Storm topologies less painful and
-deveoper-intensive.
-
-Have you ever found yourself repeating this pattern?:
-
-```java
-
-public static void main(String[] args) throws Exception {
- // logic to determine if we're running locally or not...
- // create necessary config options...
- boolean runLocal = shouldRunLocal();
- if(runLocal){
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology(name, conf, topology);
- } else {
- StormSubmitter.submitTopology(name, conf, topology);
- }
-}
-```
-
-Wouldn't something like this be easier:
-
-```bash
-storm jar mytopology.jar org.apache.storm.flux.Flux --local config.yaml
-```
-
-or:
-
-```bash
-storm jar mytopology.jar org.apache.storm.flux.Flux --remote config.yaml
-```
-
-Another pain point often mentioned is the fact that the wiring for a Topology graph is often tied up in Java code,
-and that any changes require recompilation and repackaging of the topology jar file. Flux aims to alleviate that
-pain by allowing you to package all your Storm components in a single jar, and use an external text file to define
-the layout and configuration of your topologies.
-
-## Features
-
- * Easily configure and deploy Storm topologies (Both Storm core and Microbatch API) without embedding configuration
- in your topology code
- * Support for existing topology code (see below)
- * Define Storm Core API (Spouts/Bolts) using a flexible YAML DSL
- * YAML DSL support for most Storm components (storm-kafka, storm-hdfs, storm-hbase, etc.)
- * Convenient support for multi-lang components
- * External property substitution/filtering for easily switching between configurations/environments (similar to Maven-style
- `${variable.name}` substitution)
-
-## Usage
-
-To use Flux, add it as a dependency and package all your Storm components in a fat jar, then create a YAML document
-to define your topology (see below for YAML configuration options).
-
-### Building from Source
-The easiest way to use Flux, is to add it as a Maven dependency in you project as described below.
-
-If you would like to build Flux from source and run the unit/integration tests, you will need the following installed
-on your system:
-
-* Python 2.6.x or later
-* Node.js 0.10.x or later
-
-#### Building with unit tests enabled:
-
-```
-mvn clean install
-```
-
-#### Building with unit tests disabled:
-If you would like to build Flux without installing Python or Node.js you can simply skip the unit tests:
-
-```
-mvn clean install -DskipTests=true
-```
-
-Note that if you plan on using Flux to deploy topologies to a remote cluster, you will still need to have Python
-installed since it is required by Apache Storm.
-
-
-#### Building with integration tests enabled:
-
-```
-mvn clean install -DskipIntegration=false
-```
-
-
-### Packaging with Maven
-To enable Flux for your Storm components, you need to add it as a dependency such that it's included in the Storm
-topology jar. This can be accomplished with the Maven shade plugin (preferred) or the Maven assembly plugin (not
-recommended).
-
-#### Flux Maven Dependency
-The current version of Flux is available in Maven Central at the following coordinates:
-```xml
-<dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>flux-core</artifactId>
- <version>${storm.version}</version>
-</dependency>
-```
-
-Using shell spouts and bolts requires additional Flux Wrappers library:
-```xml
-<dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>flux-wrappers</artifactId>
- <version>${storm.version}</version>
-</dependency>
-```
-
-#### Creating a Flux-Enabled Topology JAR
-The example below illustrates Flux usage with the Maven shade plugin:
-
- ```xml
-<!-- include Flux and user dependencies in the shaded jar -->
-<dependencies>
- <!-- Flux include -->
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>flux-core</artifactId>
- <version>${storm.version}</version>
- </dependency>
- <!-- Flux Wrappers include -->
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>flux-wrappers</artifactId>
- <version>${storm.version}</version>
- </dependency>
-
- <!-- add user dependencies here... -->
-
-</dependencies>
-<!-- create a fat jar that includes all dependencies -->
-<build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>1.4</version>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.storm.flux.Flux</mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
-</build>
- ```
-
-### Deploying and Running a Flux Topology
-Once your topology components are packaged with the Flux dependency, you can run different topologies either locally
-or remotely using the `storm jar` command. For example, if your fat jar is named `myTopology-0.1.0-SNAPSHOT.jar` you
-could run it locally with the command:
-
-
-```bash
-storm jar myTopology-0.1.0-SNAPSHOT.jar org.apache.storm.flux.Flux --local my_config.yaml
-
-```
-
-### Command line options
-```
-usage: storm jar <my_topology_uber_jar.jar> org.apache.storm.flux.Flux
- [options] <topology-config.yaml>
- -d,--dry-run Do not run or deploy the topology. Just
- build, validate, and print information about
- the topology.
- -e,--env-filter Perform environment variable substitution.
- Replace keys identified with `${ENV-[NAME]}`
- will be replaced with the corresponding
- `NAME` environment value
- -f,--filter <file> Perform property substitution. Use the
- specified file as a source of properties,
- and replace keys identified with {$[property
- name]} with the value defined in the
- properties file.
- -i,--inactive Deploy the topology, but do not activate it.
- -l,--local Run the topology in local mode.
- -n,--no-splash Suppress the printing of the splash screen.
- -q,--no-detail Suppress the printing of topology details.
- -r,--remote Deploy the topology to a remote cluster.
- -R,--resource Treat the supplied path as a classpath
- resource instead of a file.
- -s,--sleep <ms> When running locally, the amount of time to
- sleep (in ms.) before killing the topology
- and shutting down the local cluster.
- -z,--zookeeper <host:port> When running in local mode, use the
- ZooKeeper at the specified <host>:<port>
- instead of the in-process ZooKeeper.
- (requires Storm 0.9.3 or later)
-```
-
-**NOTE:** Flux tries to avoid command line switch collision with the `storm` command, and allows any other command line
-switches to pass through to the `storm` command.
-
-For example, you can use the `storm` command switch `-c` to override a topology configuration property. The following
-example command will run Flux and override the `nimus.host` configuration:
-
-```bash
-storm jar myTopology-0.1.0-SNAPSHOT.jar org.apache.storm.flux.Flux --remote my_config.yaml -c nimbus.host=localhost
-```
-
-### Sample output
-```
-\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2557
-\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u255a\u2588\u2588\u2557\u2588\u2588\u2554\u255d
-\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551 \u255a\u2588\u2588\u2588\u2554\u255d
-\u2588\u2588\u2554\u2550\u2550\u255d \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2554\u2588\u2588\u2557
-\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2554\u255d \u2588\u2588\u2557
-\u255a\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u2550\u2550\u2550\u2550\u255d \u255a\u2550\u255d \u255a\u2550\u255d
-+- Apache Storm -+
-+- data FLow User eXperience -+
-Version: 0.3.0
-Parsing file: /Users/hsimpson/Projects/donut_domination/storm/shell_test.yaml
----------- TOPOLOGY DETAILS ----------
-Name: shell-topology
---------------- SPOUTS ---------------
-sentence-spout[1](org.apache.storm.flux.wrappers.spouts.FluxShellSpout)
----------------- BOLTS ---------------
-splitsentence[1](org.apache.storm.flux.wrappers.bolts.FluxShellBolt)
-log[1](org.apache.storm.flux.wrappers.bolts.LogInfoBolt)
-count[1](org.apache.storm.testing.TestWordCounter)
---------------- STREAMS ---------------
-sentence-spout --SHUFFLE--> splitsentence
-splitsentence --FIELDS--> count
-count --SHUFFLE--> log
---------------------------------------
-Submitting topology: 'shell-topology' to remote cluster...
-```
-
-## YAML Configuration
-Flux topologies are defined in a YAML file that describes a topology. A Flux topology
-definition consists of the following:
-
- 1. A topology name
- 2. A list of topology "components" (named Java objects that will be made available in the environment)
- 3. **EITHER** (A DSL topology definition):
- * A list of spouts, each identified by a unique ID
- * A list of bolts, each identified by a unique ID
- * A list of "stream" objects representing a flow of tuples between spouts and bolts
- 4. **OR** (A JVM class that can produce a `org.apache.storm.generated.StormTopology` instance:
- * A `topologySource` definition.
-
-
-
-For example, here is a simple definition of a wordcount topology using the YAML DSL:
-
-```yaml
-name: "yaml-topology"
-config:
- topology.workers: 1
-
-# spout definitions
-spouts:
- - id: "spout-1"
- className: "org.apache.storm.testing.TestWordSpout"
- parallelism: 1
-
-# bolt definitions
-bolts:
- - id: "bolt-1"
- className: "org.apache.storm.testing.TestWordCounter"
- parallelism: 1
- - id: "bolt-2"
- className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt"
- parallelism: 1
-
-#stream definitions
-streams:
- - name: "spout-1 --> bolt-1" # name isn't used (placeholder for logging, UI, etc.)
- from: "spout-1"
- to: "bolt-1"
- grouping:
- type: FIELDS
- args: ["word"]
-
- - name: "bolt-1 --> bolt2"
- from: "bolt-1"
- to: "bolt-2"
- grouping:
- type: SHUFFLE
-
-
-```
-## Property Substitution/Filtering
-It's common for developers to want to easily switch between configurations, for example switching deployment between
-a development environment and a production environment. This can be accomplished by using separate YAML configuration
-files, but that approach would lead to unnecessary duplication, especially in situations where the Storm topology
-does not change, but configuration settings such as host names, ports, and parallelism paramters do.
-
-For this case, Flux offers properties filtering to allow you two externalize values to a `.properties` file and have
-them substituted before the `.yaml` file is parsed.
-
-To enable property filtering, use the `--filter` command line option and specify a `.properties` file. For example,
-if you invoked flux like so:
-
-```bash
-storm jar myTopology-0.1.0-SNAPSHOT.jar org.apache.storm.flux.Flux --local my_config.yaml --filter dev.properties
-```
-With the following `dev.properties` file:
-
-```properties
-kafka.zookeeper.hosts: localhost:2181
-```
-
-You would then be able to reference those properties by key in your `.yaml` file using `${}` syntax:
-
-```yaml
- - id: "zkHosts"
- className: "org.apache.storm.kafka.ZkHosts"
- constructorArgs:
- - "${kafka.zookeeper.hosts}"
-```
-
-In this case, Flux would replace `${kafka.zookeeper.hosts}` with `localhost:2181` before parsing the YAML contents.
-
-### Environment Variable Substitution/Filtering
-Flux also allows environment variable substitution. For example, if an environment variable named `ZK_HOSTS` if defined,
-you can reference it in a Flux YAML file with the following syntax:
-
-```
-${ENV-ZK_HOSTS}
-```
-
-## Components
-Components are essentially named object instances that are made available as configuration options for spouts and
-bolts. If you are familiar with the Spring framework, components are roughly analagous to Spring beans.
-
-Every component is identified, at a minimum, by a unique identifier (String) and a class name (String). For example,
-the following will make an instance of the `org.apache.storm.kafka.StringScheme` class available as a reference under the key
-`"stringScheme"` . This assumes the `org.apache.storm.kafka.StringScheme` has a default constructor.
-
-```yaml
-components:
- - id: "stringScheme"
- className: "org.apache.storm.kafka.StringScheme"
-```
-
-### Contructor Arguments, References, Properties and Configuration Methods
-
-####Constructor Arguments
-Arguments to a class constructor can be configured by adding a `contructorArgs` element to a components.
-`constructorArgs` is a list of objects that will be passed to the class' constructor. The following example creates an
-object by calling the constructor that takes a single string as an argument:
-
-```yaml
- - id: "zkHosts"
- className: "org.apache.storm.kafka.ZkHosts"
- constructorArgs:
- - "localhost:2181"
- - true
-```
-
-####References
-Each component instance is identified by a unique id that allows it to be used/reused by other components. To
-reference an existing component, you specify the id of the component with the `ref` tag.
-
-In the following example, a component with the id `"stringScheme"` is created, and later referenced, as a an argument
-to another component's constructor:
-
-```yaml
-components:
- - id: "stringScheme"
- className: "org.apache.storm.kafka.StringScheme"
-
- - id: "stringMultiScheme"
- className: "org.apache.storm.spout.SchemeAsMultiScheme"
- constructorArgs:
- - ref: "stringScheme" # component with id "stringScheme" must be declared above.
-```
-
-You can also reference existing components in list via specifying the id of the components with the `reflist` tag.
-The type of the reflist will be `List<Object>`, but Flux can automatically convert List to Array (also varargs),
-so you can use reflist on argument which type is `List<Type>`, or `Type[]`, or `Type...`.
-
-Please note that all components in the list must be same type.
-
-```yaml
-components:
- - id: "boundCQLStatementMapperBuilder"
- className: "org.apache.storm.cassandra.query.builder.BoundCQLStatementMapperBuilder"
- constructorArgs:
- - "INSERT INTO sink_cassandra (eventKey, driverId, truckId, driverName) VALUES (?, ?, ?, ?)"
- configMethods:
- - name: "bind"
- args:
- - reflist: ["FieldSelector-1", "FieldSelector-2", "FieldSelector-3", "FieldSelector-4"]
-```
-
-**N.B.:** References can only be used after (below) the object they point to has been declared.
-
-####Properties
-In addition to calling constructors with different arguments, Flux also allows you to configure components using
-JavaBean-like setter methods and fields declared as `public`:
-
-```yaml
- - id: "spoutConfig"
- className: "org.apache.storm.kafka.SpoutConfig"
- constructorArgs:
- # brokerHosts
- - ref: "zkHosts"
- # topic
- - "myKafkaTopic"
- # zkRoot
- - "/kafkaSpout"
- # id
- - "myId"
- properties:
- - name: "ignoreZkOffsets"
- value: true
- - name: "scheme"
- ref: "stringMultiScheme"
-```
-
-In the example above, the `properties` declaration will cause Flux to look for a public method in the `SpoutConfig` with
-the signature `setForceFromStart(boolean b)` and attempt to invoke it. If a setter method is not found, Flux will then
-look for a public instance variable with the name `ignoreZkOffsets` and attempt to set its value.
-
-References may also be used as property values.
-
-####Configuration Methods
-Conceptually, configuration methods are similar to Properties and Constructor Args -- they allow you to invoke an
-arbitrary method on an object after it is constructed. Configuration methods are useful for working with classes that
-don't expose JavaBean methods or have constructors that can fully configure the object. Common examples include classes
-that use the builder pattern for configuration/composition.
-
-The following YAML example creates a bolt and configures it by calling several methods:
-
-```yaml
-bolts:
- - id: "bolt-1"
- className: "org.apache.storm.flux.test.TestBolt"
- parallelism: 1
- configMethods:
- - name: "withFoo"
- args:
- - "foo"
- - name: "withNone"
- # no args needed, so no "args" line
- - name: "withBar"
- args:
- - "bar"
- - name: "withFooBar"
- args:
- - "foo"
- - "bar"
-```
-
-The signatures of the corresponding methods are as follows:
-
-```java
- public void withFoo(String foo);
- public void withNone(); // method with zero arguments
- public void withBar(String bar);
- public void withFooBar(String foo, String bar);
-```
-
-Arguments passed to configuration methods work much the same way as constructor arguments, and support references as
-well.
-
-### Using Java `enum`s in Contructor Arguments, References, Properties and Configuration Methods
-You can easily use Java `enum` values as arguments in a Flux YAML file, simply by referencing the name of the `enum`.
-
-For example, [Storm's HDFS module]() includes the following `enum` definition (simplified for brevity):
-
-```java
-public static enum Units {
- KB, MB, GB, TB
-}
-```
-
-And the `org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy` class has the following constructor:
-
-```java
-public FileSizeRotationPolicy(float count, Units units)
-
-```
-The following Flux `component` definition could be used to call the constructor:
-
-```yaml
- - id: "rotationPolicy"
- className: "org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy"
- constructorArgs:
- - 5.0
- - MB
-```
-
-The above definition is functionally equivalent to the following Java code:
-
-```java
-// rotate files when they reach 5MB
-FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB);
-```
-
-## Topology Config
-The `config` section is simply a map of Storm topology configuration parameters that will be passed to the
-`org.apache.storm.StormSubmitter` as an instance of the `org.apache.storm.Config` class:
-
-```yaml
-config:
- topology.workers: 4
- topology.max.spout.pending: 1000
- topology.message.timeout.secs: 30
-```
-
-# Existing Topologies
-If you have existing Storm topologies, you can still use Flux to deploy/run/test them. This feature allows you to
-leverage Flux Constructor Arguments, References, Properties, and Topology Config declarations for existing topology
-classes.
-
-The easiest way to use an existing topology class is to define
-a `getTopology()` instance method with one of the following signatures:
-
-```java
-public StormTopology getTopology(Map<String, Object> config)
-```
-or:
-
-```java
-public StormTopology getTopology(Config config)
-```
-
-You could then use the following YAML to configure your topology:
-
-```yaml
-name: "existing-topology"
-topologySource:
- className: "org.apache.storm.flux.test.SimpleTopology"
-```
-
-If the class you would like to use as a topology source has a different method name (i.e. not `getTopology`), you can
-override it:
-
-```yaml
-name: "existing-topology"
-topologySource:
- className: "org.apache.storm.flux.test.SimpleTopology"
- methodName: "getTopologyWithDifferentMethodName"
-```
-
-__N.B.:__ The specified method must accept a single argument of type `java.util.Map<String, Object>` or
-`org.apache.storm.Config`, and return a `org.apache.storm.generated.StormTopology` object.
-
-# YAML DSL
-## Spouts and Bolts
-Spout and Bolts are configured in their own respective section of the YAML configuration. Spout and Bolt definitions
-are extensions to the `component` definition that add a `parallelism` parameter that sets the parallelism for a
-component when the topology is deployed.
-
-Because spout and bolt definitions extend `component` they support constructor arguments, references, and properties as
-well.
-
-Shell spout example:
-
-```yaml
-spouts:
- - id: "sentence-spout"
- className: "org.apache.storm.flux.wrappers.spouts.FluxShellSpout"
- # shell spout constructor takes 2 arguments: String[], String[]
- constructorArgs:
- # command line
- - ["node", "randomsentence.js"]
- # output fields
- - ["word"]
- parallelism: 1
-```
-
-Kafka spout example:
-
-```yaml
-components:
- - id: "stringScheme"
- className: "org.apache.storm.kafka.StringScheme"
-
- - id: "stringMultiScheme"
- className: "org.apache.storm.spout.SchemeAsMultiScheme"
- constructorArgs:
- - ref: "stringScheme"
-
- - id: "zkHosts"
- className: "org.apache.storm.kafka.ZkHosts"
- constructorArgs:
- - "localhost:2181"
-
-# Alternative kafka config
-# - id: "kafkaConfig"
-# className: "org.apache.storm.kafka.KafkaConfig"
-# constructorArgs:
-# # brokerHosts
-# - ref: "zkHosts"
-# # topic
-# - "myKafkaTopic"
-# # clientId (optional)
-# - "myKafkaClientId"
-
- - id: "spoutConfig"
- className: "org.apache.storm.kafka.SpoutConfig"
- constructorArgs:
- # brokerHosts
- - ref: "zkHosts"
- # topic
- - "myKafkaTopic"
- # zkRoot
- - "/kafkaSpout"
- # id
- - "myId"
- properties:
- - name: "ignoreZkOffsets"
- value: true
- - name: "scheme"
- ref: "stringMultiScheme"
-
-config:
- topology.workers: 1
-
-# spout definitions
-spouts:
- - id: "kafka-spout"
- className: "org.apache.storm.kafka.KafkaSpout"
- constructorArgs:
- - ref: "spoutConfig"
-
-```
-
-Bolt Examples:
-
-```yaml
-# bolt definitions
-bolts:
- - id: "splitsentence"
- className: "org.apache.storm.flux.wrappers.bolts.FluxShellBolt"
- constructorArgs:
- # command line
- - ["python", "splitsentence.py"]
- # output fields
- - ["word"]
- parallelism: 1
- # ...
-
- - id: "log"
- className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt"
- parallelism: 1
- # ...
-
- - id: "count"
- className: "org.apache.storm.testing.TestWordCounter"
- parallelism: 1
- # ...
-```
-## Streams and Stream Groupings
-Streams in Flux are represented as a list of connections (Graph edges, data flow, etc.) between the Spouts and Bolts in
-a topology, with an associated Grouping definition.
-
-A Stream definition has the following properties:
-
-**`name`:** A name for the connection (optional, currently unused)
-
-**`from`:** The `id` of a Spout or Bolt that is the source (publisher)
-
-**`to`:** The `id` of a Spout or Bolt that is the destination (subscriber)
-
-**`grouping`:** The stream grouping definition for the Stream
-
-A Grouping definition has the following properties:
-
-**`type`:** The type of grouping. One of `ALL`,`CUSTOM`,`DIRECT`,`SHUFFLE`,`LOCAL_OR_SHUFFLE`,`FIELDS`,`GLOBAL`, or `NONE`.
-
-**`streamId`:** The Storm stream ID (Optional. If unspecified will use the default stream)
-
-**`args`:** For the `FIELDS` grouping, a list of field names.
-
-**`customClass`** For the `CUSTOM` grouping, a definition of custom grouping class instance
-
-The `streams` definition example below sets up a topology with the following wiring:
-
-```
- kafka-spout --> splitsentence --> count --> log
-```
-
-
-```yaml
-#stream definitions
-# stream definitions define connections between spouts and bolts.
-# note that such connections can be cyclical
-# custom stream groupings are also supported
-
-streams:
- - name: "kafka --> split" # name isn't used (placeholder for logging, UI, etc.)
- from: "kafka-spout"
- to: "splitsentence"
- grouping:
- type: SHUFFLE
-
- - name: "split --> count"
- from: "splitsentence"
- to: "count"
- grouping:
- type: FIELDS
- args: ["word"]
-
- - name: "count --> log"
- from: "count"
- to: "log"
- grouping:
- type: SHUFFLE
-```
-
-### Custom Stream Groupings
-Custom stream groupings are defined by setting the grouping type to `CUSTOM` and defining a `customClass` parameter
-that tells Flux how to instantiate the custom class. The `customClass` definition extends `component`, so it supports
-constructor arguments, references, and properties as well.
-
-The example below creates a Stream with an instance of the `org.apache.storm.testing.NGrouping` custom stream grouping
-class.
-
-```yaml
- - name: "bolt-1 --> bolt2"
- from: "bolt-1"
- to: "bolt-2"
- grouping:
- type: CUSTOM
- customClass:
- className: "org.apache.storm.testing.NGrouping"
- constructorArgs:
- - 1
-```
-
-## Includes and Overrides
-Flux allows you to include the contents of other YAML files, and have them treated as though they were defined in the
-same file. Includes may be either files, or classpath resources.
-
-Includes are specified as a list of maps:
-
-```yaml
-includes:
- - resource: false
- file: "src/test/resources/configs/shell_test.yaml"
- override: false
-```
-
-If the `resource` property is set to `true`, the include will be loaded as a classpath resource from the value of the
-`file` attribute, otherwise it will be treated as a regular file.
-
-The `override` property controls how includes affect the values defined in the current file. If `override` is set to
-`true`, values in the included file will replace values in the current file being parsed. If `override` is set to
-`false`, values in the current file being parsed will take precedence, and the parser will refuse to replace them.
-
-**N.B.:** Includes are not yet recursive. Includes from included files will be ignored.
-
-
-## Basic Word Count Example
-
-This example uses a spout implemented in JavaScript, a bolt implemented in Python, and a bolt implemented in Java
-
-Topology YAML config:
-
-```yaml
----
-name: "shell-topology"
-config:
- topology.workers: 1
-
-# spout definitions
-spouts:
- - id: "sentence-spout"
- className: "org.apache.storm.flux.wrappers.spouts.FluxShellSpout"
- # shell spout constructor takes 2 arguments: String[], String[]
- constructorArgs:
- # command line
- - ["node", "randomsentence.js"]
- # output fields
- - ["word"]
- parallelism: 1
-
-# bolt definitions
-bolts:
- - id: "splitsentence"
- className: "org.apache.storm.flux.wrappers.bolts.FluxShellBolt"
- constructorArgs:
- # command line
- - ["python", "splitsentence.py"]
- # output fields
- - ["word"]
- parallelism: 1
-
- - id: "log"
- className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt"
- parallelism: 1
-
- - id: "count"
- className: "org.apache.storm.testing.TestWordCounter"
- parallelism: 1
-
-#stream definitions
-# stream definitions define connections between spouts and bolts.
-# note that such connections can be cyclical
-# custom stream groupings are also supported
-
-streams:
- - name: "spout --> split" # name isn't used (placeholder for logging, UI, etc.)
- from: "sentence-spout"
- to: "splitsentence"
- grouping:
- type: SHUFFLE
-
- - name: "split --> count"
- from: "splitsentence"
- to: "count"
- grouping:
- type: FIELDS
- args: ["word"]
-
- - name: "count --> log"
- from: "count"
- to: "log"
- grouping:
- type: SHUFFLE
-```
-
-
-## Micro-Batching (Trident) API Support
-Currenty, the Flux YAML DSL only supports the Core Storm API, but support for Storm's micro-batching API is planned.
-
-To use Flux with a Trident topology, define a topology getter method and reference it in your YAML config:
-
-```yaml
-name: "my-trident-topology"
-
-config:
- topology.workers: 1
-
-topologySource:
- className: "org.apache.storm.flux.test.TridentTopologySource"
- # Flux will look for "getTopology", this will override that.
- methodName: "getTopologyWithDifferentMethodName"
-```
-
-## Committer Sponsors
-
- * P. Taylor Goetz ([ptgoetz@apache.org](mailto:ptgoetz@apache.org))
http://git-wip-us.apache.org/repos/asf/storm/blob/e9d78338/external/flux/flux-core/pom.xml
----------------------------------------------------------------------
diff --git a/external/flux/flux-core/pom.xml b/external/flux/flux-core/pom.xml
deleted file mode 100644
index 1cf4a9b..0000000
--- a/external/flux/flux-core/pom.xml
+++ /dev/null
@@ -1,112 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.storm</groupId>
- <artifactId>flux</artifactId>
- <version>2.0.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>flux-core</artifactId>
- <packaging>jar</packaging>
-
- <name>flux-core</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>flux-wrappers</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-kafka</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-hdfs</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-hbase</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.yaml</groupId>
- <artifactId>snakeyaml</artifactId>
- </dependency>
- </dependencies>
- <build>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- <filtering>true</filtering>
- </resource>
- </resources>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <configuration>
- <createDependencyReducedPom>true</createDependencyReducedPom>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.sf</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.dsa</exclude>
- <exclude>META-INF/*.RSA</exclude>
- <exclude>META-INF/*.rsa</exclude>
- <exclude>META-INF/*.EC</exclude>
- <exclude>META-INF/*.ec</exclude>
- <exclude>META-INF/MSFTSIG.SF</exclude>
- <exclude>META-INF/MSFTSIG.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.storm.flux.Flux</mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/storm/blob/e9d78338/external/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java
----------------------------------------------------------------------
diff --git a/external/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java b/external/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java
deleted file mode 100644
index 982765f..0000000
--- a/external/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.storm.flux;
-
-import org.apache.storm.Config;
-import org.apache.storm.LocalCluster;
-import org.apache.storm.LocalCluster.LocalTopology;
-import org.apache.storm.StormSubmitter;
-import org.apache.storm.generated.StormTopology;
-import org.apache.storm.generated.SubmitOptions;
-import org.apache.storm.generated.TopologyInitialStatus;
-import org.apache.storm.utils.Utils;
-import org.apache.commons.cli.*;
-import org.apache.storm.flux.model.*;
-import org.apache.storm.flux.parser.FluxParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.*;
-import java.util.Map;
-import java.util.Properties;
-
-/**
- * Flux entry point.
- *
- */
-public class Flux {
- private static final Logger LOG = LoggerFactory.getLogger(Flux.class);
-
- private static final Long DEFAULT_LOCAL_SLEEP_TIME = 60000l;
-
- private static final Long DEFAULT_ZK_PORT = 2181l;
-
- private static final String OPTION_LOCAL = "local";
- private static final String OPTION_REMOTE = "remote";
- private static final String OPTION_RESOURCE = "resource";
- private static final String OPTION_SLEEP = "sleep";
- private static final String OPTION_DRY_RUN = "dry-run";
- private static final String OPTION_NO_DETAIL = "no-detail";
- private static final String OPTION_NO_SPLASH = "no-splash";
- private static final String OPTION_INACTIVE = "inactive";
- private static final String OPTION_ZOOKEEPER = "zookeeper";
- private static final String OPTION_FILTER = "filter";
- private static final String OPTION_ENV_FILTER = "env-filter";
-
- public static void main(String[] args) throws Exception {
- Options options = new Options();
-
- options.addOption(option(0, "l", OPTION_LOCAL, "Run the topology in local mode."));
-
- options.addOption(option(0, "r", OPTION_REMOTE, "Deploy the topology to a remote cluster."));
-
- options.addOption(option(0, "R", OPTION_RESOURCE, "Treat the supplied path as a classpath resource instead of a file."));
-
- options.addOption(option(1, "s", OPTION_SLEEP, "ms", "When running locally, the amount of time to sleep (in ms.) " +
- "before killing the topology and shutting down the local cluster."));
-
- options.addOption(option(0, "d", OPTION_DRY_RUN, "Do not run or deploy the topology. Just build, validate, " +
- "and print information about the topology."));
-
- options.addOption(option(0, "q", OPTION_NO_DETAIL, "Suppress the printing of topology details."));
-
- options.addOption(option(0, "n", OPTION_NO_SPLASH, "Suppress the printing of the splash screen."));
-
- options.addOption(option(0, "i", OPTION_INACTIVE, "Deploy the topology, but do not activate it."));
-
- options.addOption(option(1, "z", OPTION_ZOOKEEPER, "host:port", "When running in local mode, use the ZooKeeper at the " +
- "specified <host>:<port> instead of the in-process ZooKeeper. (requires Storm 0.9.3 or later)"));
-
- options.addOption(option(1, "f", OPTION_FILTER, "file", "Perform property substitution. Use the specified file " +
- "as a source of properties, and replace keys identified with {$[property name]} with the value defined " +
- "in the properties file."));
-
- options.addOption(option(0, "e", OPTION_ENV_FILTER, "Perform environment variable substitution. Replace keys" +
- "identified with `${ENV-[NAME]}` will be replaced with the corresponding `NAME` environment value"));
-
- CommandLineParser parser = new BasicParser();
- CommandLine cmd = parser.parse(options, args);
-
- if (cmd.getArgs().length != 1) {
- usage(options);
- System.exit(1);
- }
- runCli(cmd);
- }
-
- private static Option option(int argCount, String shortName, String longName, String description){
- return option(argCount, shortName, longName, longName, description);
- }
-
- private static Option option(int argCount, String shortName, String longName, String argName, String description){
- Option option = OptionBuilder.hasArgs(argCount)
- .withArgName(argName)
- .withLongOpt(longName)
- .withDescription(description)
- .create(shortName);
- return option;
- }
-
- private static void usage(Options options) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("storm jar <my_topology_uber_jar.jar> " +
- Flux.class.getName() +
- " [options] <topology-config.yaml>", options);
- }
-
- private static void runCli(CommandLine cmd)throws Exception {
- if(!cmd.hasOption(OPTION_NO_SPLASH)) {
- printSplash();
- }
-
- boolean dumpYaml = cmd.hasOption("dump-yaml");
-
- TopologyDef topologyDef = null;
- String filePath = (String)cmd.getArgList().get(0);
-
- // TODO conditionally load properties from a file our resource
- String filterProps = null;
- if(cmd.hasOption(OPTION_FILTER)){
- filterProps = cmd.getOptionValue(OPTION_FILTER);
- }
-
-
- boolean envFilter = cmd.hasOption(OPTION_ENV_FILTER);
- if(cmd.hasOption(OPTION_RESOURCE)){
- printf("Parsing classpath resource: %s", filePath);
- topologyDef = FluxParser.parseResource(filePath, dumpYaml, true, filterProps, envFilter);
- } else {
- printf("Parsing file: %s",
- new File(filePath).getAbsolutePath());
- topologyDef = FluxParser.parseFile(filePath, dumpYaml, true, filterProps, envFilter);
- }
-
-
- String topologyName = topologyDef.getName();
- // merge contents of `config` into topology config
- Config conf = FluxBuilder.buildConfig(topologyDef);
- ExecutionContext context = new ExecutionContext(topologyDef, conf);
- StormTopology topology = FluxBuilder.buildTopology(context);
-
- if(!cmd.hasOption(OPTION_NO_DETAIL)){
- printTopologyInfo(context);
- }
-
- if(!cmd.hasOption(OPTION_DRY_RUN)) {
- if (cmd.hasOption(OPTION_REMOTE)) {
- LOG.info("Running remotely...");
- // should the topology be active or inactive
- SubmitOptions submitOptions = null;
- if(cmd.hasOption(OPTION_INACTIVE)){
- LOG.info("Deploying topology in an INACTIVE state...");
- submitOptions = new SubmitOptions(TopologyInitialStatus.INACTIVE);
- } else {
- LOG.info("Deploying topology in an ACTIVE state...");
- submitOptions = new SubmitOptions(TopologyInitialStatus.ACTIVE);
- }
- StormSubmitter.submitTopology(topologyName, conf, topology, submitOptions, null);
- } else {
- LOG.info("Running in local mode...");
-
- String sleepStr = cmd.getOptionValue(OPTION_SLEEP);
- Long sleepTime = DEFAULT_LOCAL_SLEEP_TIME;
- if (sleepStr != null) {
- sleepTime = Long.parseLong(sleepStr);
- }
- LOG.debug("Sleep time: {}", sleepTime);
- LocalCluster cluster = null;
-
- // in-process or external zookeeper
- if(cmd.hasOption(OPTION_ZOOKEEPER)){
- String zkStr = cmd.getOptionValue(OPTION_ZOOKEEPER);
- LOG.info("Using ZooKeeper at '{}' instead of in-process one.", zkStr);
- long zkPort = DEFAULT_ZK_PORT;
- String zkHost = null;
- if(zkStr.contains(":")){
- String[] hostPort = zkStr.split(":");
- zkHost = hostPort[0];
- zkPort = hostPort.length > 1 ? Long.parseLong(hostPort[1]) : DEFAULT_ZK_PORT;
-
- } else {
- zkHost = zkStr;
- }
- // the following constructor is only available in 0.9.3 and later
- try {
- cluster = new LocalCluster(zkHost, zkPort);
- } catch (NoSuchMethodError e){
- LOG.error("The --zookeeper option can only be used with Apache Storm 0.9.3 and later.");
- System.exit(1);
- }
- } else {
- cluster = new LocalCluster();
- }
- try (LocalTopology topo = cluster.submitTopology(topologyName, conf, topology)) {
- Utils.sleep(sleepTime);
- } finally {
- cluster.shutdown();
- }
- }
- }
- }
-
- static void printTopologyInfo(ExecutionContext ctx){
- TopologyDef t = ctx.getTopologyDef();
- if(t.isDslTopology()) {
- print("---------- TOPOLOGY DETAILS ----------");
-
- printf("Topology Name: %s", t.getName());
- print("--------------- SPOUTS ---------------");
- for (SpoutDef s : t.getSpouts()) {
- printf("%s [%d] (%s)", s.getId(), s.getParallelism(), s.getClassName());
- }
- print("---------------- BOLTS ---------------");
- for (BoltDef b : t.getBolts()) {
- printf("%s [%d] (%s)", b.getId(), b.getParallelism(), b.getClassName());
- }
-
- print("--------------- STREAMS ---------------");
- for (StreamDef sd : t.getStreams()) {
- printf("%s --%s--> %s", sd.getFrom(), sd.getGrouping().getType(), sd.getTo());
- }
- print("--------------------------------------");
- }
- }
-
- // save a little typing
- private static void printf(String format, Object... args){
- print(String.format(format, args));
- }
-
- private static void print(String string){
- System.out.println(string);
- }
-
- private static void printSplash() throws IOException {
- // banner
- InputStream is = Flux.class.getResourceAsStream("/splash.txt");
- if(is != null){
- InputStreamReader isr = new InputStreamReader(is, "UTF-8");
- BufferedReader br = new BufferedReader(isr);
- String line = null;
- while((line = br.readLine()) != null){
- System.out.println(line);
- }
- }
- }
-}