You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2015/12/01 19:09:33 UTC
[49/50] [abbrv] storm git commit: Merge branch 'master' into
STORM-1040
Merge branch 'master' into STORM-1040
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/31b49594
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/31b49594
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/31b49594
Branch: refs/heads/STORM-1040
Commit: 31b4959475f8d09d2e9dd90856a1941adcc94e8c
Parents: 4b1062e 0acc1ce
Author: Haohui Mai <wh...@apache.org>
Authored: Mon Nov 30 23:42:56 2015 -0800
Committer: Haohui Mai <wh...@apache.org>
Committed: Mon Nov 30 23:52:40 2015 -0800
----------------------------------------------------------------------
.gitignore | 3 +
.travis.yml | 17 +-
CHANGELOG.md | 115 +-
DEVELOPER.md | 35 +-
DISCLAIMER | 10 -
LICENSE | 41 +
README.markdown | 11 +
STORM-UI-REST-API.md | 707 -
bin/flight.bash | 154 +
bin/storm | 22 +-
bin/storm-config.cmd | 10 +-
bin/storm.py | 115 +-
conf/defaults.yaml | 45 +-
conf/storm.yaml.example | 2 +-
dev-tools/storm-merge.py | 2 +-
dev-tools/travis/ratprint.py | 26 +
dev-tools/travis/travis-install.sh | 9 +-
dev-tools/travis/travis-script.sh | 15 +-
docs/documentation/Documentation.md | 4 +
docs/documentation/FAQ.md | 2 +-
docs/documentation/Log-Search.md | 14 +
.../Message-passing-implementation.md | 34 +-
docs/documentation/Pacemaker.md | 108 +
.../documentation/Setting-up-a-Storm-cluster.md | 19 +
docs/documentation/Windowing.md | 144 +
.../documentation/dynamic-log-level-settings.md | 41 +
docs/documentation/dynamic-worker-profiling.md | 29 +
.../images/dynamic_log_level_settings_1.png | Bin 0 -> 93689 bytes
.../images/dynamic_log_level_settings_2.png | Bin 0 -> 78785 bytes
.../images/dynamic_profiling_debugging_1.png | Bin 0 -> 93635 bytes
.../images/dynamic_profiling_debugging_2.png | Bin 0 -> 138120 bytes
.../images/dynamic_profiling_debugging_3.png | Bin 0 -> 96974 bytes
docs/documentation/images/search-a-topology.png | Bin 0 -> 671031 bytes
.../images/search-for-a-single-worker-log.png | Bin 0 -> 736579 bytes
.../storm-metrics-profiling-internal-actions.md | 70 +
docs/documentation/ui-rest-api.md | 996 +
docs/images/viewing_metrics_with_VisualVM.png | Bin 0 -> 225100 bytes
examples/storm-starter/pom.xml | 17 +-
.../storm/starter/FastWordCountTopology.java | 198 +
.../jvm/storm/starter/InOrderDeliveryTest.java | 175 +
.../storm/starter/MultipleLoggerTopology.java | 105 +
.../starter/ResourceAwareExampleTopology.java | 101 +
.../storm/starter/SlidingWindowTopology.java | 185 +
.../jvm/storm/starter/ThroughputVsLatency.java | 432 +
.../starter/trident/TridentKafkaWordCount.java | 15 +-
.../bolt/IntermediateRankingsBoltTest.java | 2 +-
.../starter/bolt/RollingCountBoltTest.java | 2 +-
.../starter/bolt/TotalRankingsBoltTest.java | 2 +-
.../storm/starter/tools/MockTupleHelpers.java | 40 -
external/flux/README.md | 4 +
external/flux/flux-core/pom.xml | 1 -
.../main/java/org/apache/storm/flux/Flux.java | 3 +-
.../java/org/apache/storm/flux/FluxBuilder.java | 55 +-
.../org/apache/storm/flux/model/ObjectDef.java | 2 +
.../org/apache/storm/flux/test/TestBolt.java | 8 +
.../resources/configs/config-methods-test.yaml | 2 +
external/flux/flux-examples/README.md | 9 +
external/flux/flux-examples/pom.xml | 1 -
.../storm/flux/examples/TestPrintBolt.java | 39 +
.../storm/flux/examples/TestWindowBolt.java | 47 +
.../src/main/resources/simple_windowing.yaml | 69 +
.../storm/sql/compiler/TestCompilerUtils.java | 17 +
.../org/apache/storm/sql/kafka/JsonScheme.java | 6 +-
.../storm/sql/kafka/TestJsonRepresentation.java | 2 +-
.../sql/kafka/TestKafkaDataSourcesProvider.java | 12 +-
external/storm-elasticsearch/pom.xml | 5 +
external/storm-eventhubs/pom.xml | 1 -
external/storm-hbase/README.md | 10 +
.../storm/hbase/bolt/AbstractHBaseBolt.java | 2 +
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 75 +-
external/storm-hdfs/README.md | 33 +
external/storm-hdfs/pom.xml | 71 +-
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 124 +
.../storm/hdfs/bolt/AvroGenericRecordBolt.java | 145 +
.../org/apache/storm/hdfs/bolt/HdfsBolt.java | 51 +-
.../storm/hdfs/bolt/SequenceFileBolt.java | 42 +-
.../ha/codedistributor/HDFSCodeDistributor.java | 17 +
.../hdfs/bolt/AvroGenericRecordBoltTest.java | 220 +
.../apache/storm/hdfs/bolt/TestHdfsBolt.java | 258 +
.../storm/hdfs/bolt/TestSequenceFileBolt.java | 186 +
.../storm/hdfs/trident/HdfsStateTest.java | 17 +
external/storm-hive/pom.xml | 7 +
.../org/apache/storm/hive/bolt/HiveBolt.java | 9 +
.../apache/storm/hive/bolt/TestHiveBolt.java | 56 +-
.../storm/jdbc/bolt/AbstractJdbcBolt.java | 2 +
.../apache/storm/jdbc/bolt/JdbcInsertBolt.java | 9 +
.../apache/storm/jdbc/bolt/JdbcLookupBolt.java | 5 +
.../jdbc/mapper/SimpleJdbcLookupMapper.java | 3 +
.../storm/jdbc/mapper/SimpleJdbcMapper.java | 5 +
.../storm/jdbc/bolt/JdbcInsertBoltTest.java | 71 +
.../storm/jdbc/bolt/JdbcLookupBoltTest.java | 59 +
external/storm-kafka/README.md | 86 +-
external/storm-kafka/pom.xml | 5 +
.../jvm/storm/kafka/DynamicBrokersReader.java | 97 +-
.../kafka/DynamicPartitionConnections.java | 20 +-
.../src/jvm/storm/kafka/KafkaConfig.java | 3 +-
.../src/jvm/storm/kafka/KafkaSpout.java | 34 +-
.../src/jvm/storm/kafka/KafkaUtils.java | 95 +-
.../src/jvm/storm/kafka/KeyValueScheme.java | 5 +-
.../kafka/KeyValueSchemeAsMultiScheme.java | 5 +-
.../jvm/storm/kafka/MessageMetadataScheme.java | 27 +
.../MessageMetadataSchemeAsMultiScheme.java | 41 +
.../src/jvm/storm/kafka/Partition.java | 26 +-
.../src/jvm/storm/kafka/PartitionManager.java | 47 +-
.../src/jvm/storm/kafka/StaticCoordinator.java | 11 +-
.../jvm/storm/kafka/StringKeyValueScheme.java | 3 +-
.../kafka/StringMessageAndMetadataScheme.java | 43 +
.../storm/kafka/StringMultiSchemeWithTopic.java | 48 +
.../src/jvm/storm/kafka/StringScheme.java | 20 +-
.../src/jvm/storm/kafka/ZkCoordinator.java | 2 +-
.../src/jvm/storm/kafka/bolt/KafkaBolt.java | 13 +-
.../jvm/storm/kafka/trident/Coordinator.java | 7 +-
.../trident/GlobalPartitionInformation.java | 26 +-
.../jvm/storm/kafka/trident/IBrokerReader.java | 7 +-
.../kafka/trident/OpaqueTridentKafkaSpout.java | 9 +-
.../storm/kafka/trident/StaticBrokerReader.java | 23 +-
.../trident/TransactionalTridentKafkaSpout.java | 4 +-
.../kafka/trident/TridentKafkaEmitter.java | 48 +-
.../storm/kafka/trident/TridentKafkaState.java | 33 +-
.../kafka/trident/TridentKafkaStateFactory.java | 10 +-
.../jvm/storm/kafka/trident/ZkBrokerReader.java | 20 +-
.../storm/kafka/DynamicBrokersReaderTest.java | 114 +-
.../src/test/storm/kafka/KafkaUtilsTest.java | 112 +-
.../storm/kafka/StringKeyValueSchemeTest.java | 17 +-
.../src/test/storm/kafka/TestStringScheme.java | 40 +
.../src/test/storm/kafka/TestUtils.java | 28 +-
.../src/test/storm/kafka/TridentKafkaTest.java | 13 +-
.../test/storm/kafka/TridentKafkaTopology.java | 33 +-
.../src/test/storm/kafka/ZkCoordinatorTest.java | 8 +-
.../test/storm/kafka/bolt/KafkaBoltTest.java | 19 +-
external/storm-metrics/pom.xml | 107 +
.../metrics/hdrhistogram/HistogramMetric.java | 79 +
.../apache/storm/metrics/sigar/CPUMetric.java | 60 +
external/storm-solr/pom.xml | 21 +-
log4j2/cluster.xml | 42 +-
log4j2/worker.xml | 22 +-
pom.xml | 290 +-
storm-core/pom.xml | 327 +-
storm-core/src/clj/backtype/storm/cluster.clj | 341 +-
.../cluster_state/zookeeper_state_factory.clj | 157 +
.../clj/backtype/storm/command/healthcheck.clj | 88 +
.../clj/backtype/storm/command/heartbeats.clj | 52 +
.../clj/backtype/storm/command/kill_workers.clj | 33 +
.../backtype/storm/command/set_log_level.clj | 75 +
storm-core/src/clj/backtype/storm/config.clj | 112 +-
storm-core/src/clj/backtype/storm/converter.clj | 73 +-
.../backtype/storm/daemon/builtin_metrics.clj | 84 +-
.../src/clj/backtype/storm/daemon/common.clj | 42 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 46 +-
.../src/clj/backtype/storm/daemon/executor.clj | 273 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 1060 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 652 +-
.../clj/backtype/storm/daemon/supervisor.clj | 283 +-
.../src/clj/backtype/storm/daemon/task.clj | 24 +-
.../src/clj/backtype/storm/daemon/worker.clj | 270 +-
storm-core/src/clj/backtype/storm/disruptor.clj | 53 +-
.../src/clj/backtype/storm/local_state.clj | 44 +-
storm-core/src/clj/backtype/storm/log.clj | 12 +-
.../src/clj/backtype/storm/messaging/loader.clj | 76 +-
.../src/clj/backtype/storm/messaging/local.clj | 56 +-
storm-core/src/clj/backtype/storm/stats.clj | 1519 +-
storm-core/src/clj/backtype/storm/testing.clj | 34 +-
storm-core/src/clj/backtype/storm/timer.clj | 20 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 1356 +-
.../src/clj/backtype/storm/ui/helpers.clj | 77 +-
storm-core/src/clj/backtype/storm/util.clj | 82 +-
.../org/apache/storm/pacemaker/pacemaker.clj | 237 +
.../storm/pacemaker/pacemaker_state_factory.clj | 124 +
.../src/dev/logviewer-search-context-tests.log | 1 +
.../dev/logviewer-search-context-tests.log.gz | Bin 0 -> 72 bytes
storm-core/src/dev/small-worker.log | 1 +
storm-core/src/dev/test-3072.log | 3 +
storm-core/src/dev/test-worker.log | 380 +
storm-core/src/genthrift.sh | 2 +-
storm-core/src/jvm/backtype/storm/Config.java | 868 +-
.../jvm/backtype/storm/ConfigValidation.java | 375 -
.../src/jvm/backtype/storm/LogWriter.java | 2 +-
.../src/jvm/backtype/storm/StormSubmitter.java | 55 +-
.../backtype/storm/cluster/ClusterState.java | 208 +
.../storm/cluster/ClusterStateContext.java | 41 +
.../storm/cluster/ClusterStateFactory.java | 28 +
.../storm/cluster/ClusterStateListener.java | 22 +
.../backtype/storm/cluster/ConnectionState.java | 24 +
.../jvm/backtype/storm/cluster/DaemonType.java | 27 +
.../storm/codedistributor/ICodeDistributor.java | 17 +
.../LocalFileSystemCodeDistributor.java | 17 +
.../storm/coordination/BatchBoltExecutor.java | 4 +-
.../storm/coordination/CoordinatedBolt.java | 16 +-
.../storm/drpc/DRPCInvocationsClient.java | 5 +-
.../src/jvm/backtype/storm/drpc/DRPCSpout.java | 10 +-
.../src/jvm/backtype/storm/drpc/JoinResult.java | 8 +-
.../storm/generated/AlreadyAliveException.java | 4 +-
.../backtype/storm/generated/Assignment.java | 380 +-
.../storm/generated/AuthorizationException.java | 4 +-
.../src/jvm/backtype/storm/generated/Bolt.java | 4 +-
.../storm/generated/BoltAggregateStats.java | 704 +
.../jvm/backtype/storm/generated/BoltStats.java | 444 +-
.../storm/generated/ClusterSummary.java | 221 +-
.../storm/generated/ClusterWorkerHeartbeat.java | 60 +-
.../storm/generated/CommonAggregateStats.java | 902 +
.../generated/ComponentAggregateStats.java | 752 +
.../storm/generated/ComponentCommon.java | 6 +-
.../storm/generated/ComponentObject.java | 2 +-
.../storm/generated/ComponentPageInfo.java | 2194 ++
.../backtype/storm/generated/ComponentType.java | 62 +
.../backtype/storm/generated/Credentials.java | 48 +-
.../storm/generated/DRPCExecutionException.java | 4 +-
.../backtype/storm/generated/DRPCRequest.java | 4 +-
.../backtype/storm/generated/DebugOptions.java | 8 +-
.../storm/generated/DistributedRPC.java | 4 +-
.../generated/DistributedRPCInvocations.java | 4 +-
.../jvm/backtype/storm/generated/ErrorInfo.java | 8 +-
.../storm/generated/ExecutorAggregateStats.java | 526 +
.../backtype/storm/generated/ExecutorInfo.java | 8 +-
.../storm/generated/ExecutorSpecificStats.java | 2 +-
.../backtype/storm/generated/ExecutorStats.java | 174 +-
.../storm/generated/ExecutorSummary.java | 8 +-
.../storm/generated/GetInfoOptions.java | 4 +-
.../storm/generated/GlobalStreamId.java | 4 +-
.../jvm/backtype/storm/generated/Grouping.java | 2 +-
.../generated/HBAuthorizationException.java | 406 +
.../storm/generated/HBExecutionException.java | 406 +
.../jvm/backtype/storm/generated/HBMessage.java | 636 +
.../backtype/storm/generated/HBMessageData.java | 640 +
.../jvm/backtype/storm/generated/HBNodes.java | 461 +
.../jvm/backtype/storm/generated/HBPulse.java | 522 +
.../jvm/backtype/storm/generated/HBRecords.java | 466 +
.../storm/generated/HBServerMessageType.java | 113 +
.../generated/InvalidTopologyException.java | 4 +-
.../backtype/storm/generated/JavaObject.java | 4 +-
.../backtype/storm/generated/JavaObjectArg.java | 2 +-
.../backtype/storm/generated/KillOptions.java | 6 +-
.../storm/generated/LSApprovedWorkers.java | 48 +-
.../generated/LSSupervisorAssignments.java | 52 +-
.../storm/generated/LSSupervisorId.java | 4 +-
.../backtype/storm/generated/LSTopoHistory.java | 805 +
.../storm/generated/LSTopoHistoryList.java | 460 +
.../storm/generated/LSWorkerHeartbeat.java | 44 +-
.../storm/generated/LocalAssignment.java | 157 +-
.../storm/generated/LocalStateData.java | 52 +-
.../jvm/backtype/storm/generated/LogConfig.java | 475 +
.../jvm/backtype/storm/generated/LogLevel.java | 836 +
.../storm/generated/LogLevelAction.java | 65 +
.../jvm/backtype/storm/generated/Nimbus.java | 18163 ++++++++++++-----
.../backtype/storm/generated/NimbusSummary.java | 10 +-
.../jvm/backtype/storm/generated/NodeInfo.java | 36 +-
.../storm/generated/NotAliveException.java | 4 +-
.../backtype/storm/generated/NullStruct.java | 4 +-
.../storm/generated/NumErrorsChoice.java | 2 +-
.../backtype/storm/generated/ProfileAction.java | 74 +
.../storm/generated/ProfileRequest.java | 631 +
.../storm/generated/RebalanceOptions.java | 52 +-
.../storm/generated/ShellComponent.java | 4 +-
.../storm/generated/SpecificAggregateStats.java | 387 +
.../storm/generated/SpoutAggregateStats.java | 407 +
.../jvm/backtype/storm/generated/SpoutSpec.java | 4 +-
.../backtype/storm/generated/SpoutStats.java | 256 +-
.../storm/generated/StateSpoutSpec.java | 4 +-
.../jvm/backtype/storm/generated/StormBase.java | 100 +-
.../backtype/storm/generated/StormTopology.java | 255 +-
.../backtype/storm/generated/StreamInfo.java | 6 +-
.../backtype/storm/generated/SubmitOptions.java | 4 +-
.../storm/generated/SupervisorInfo.java | 282 +-
.../storm/generated/SupervisorSummary.java | 374 +-
.../storm/generated/ThriftSerializedObject.java | 4 +-
.../storm/generated/TopologyActionOptions.java | 2 +-
.../storm/generated/TopologyHistoryInfo.java | 461 +
.../backtype/storm/generated/TopologyInfo.java | 774 +-
.../storm/generated/TopologyInitialStatus.java | 2 +-
.../storm/generated/TopologyPageInfo.java | 2597 +++
.../backtype/storm/generated/TopologyStats.java | 1094 +
.../storm/generated/TopologyStatus.java | 2 +-
.../storm/generated/TopologySummary.java | 618 +-
.../storm/generated/WorkerResources.java | 605 +
.../src/jvm/backtype/storm/grouping/Load.java | 77 +
.../grouping/LoadAwareCustomStreamGrouping.java | 24 +
.../grouping/LoadAwareShuffleGrouping.java | 76 +
.../backtype/storm/grouping/LoadMapping.java | 64 +
.../storm/grouping/PartialKeyGrouping.java | 5 +-
.../storm/grouping/ShuffleGrouping.java | 65 +
.../backtype/storm/hooks/BaseWorkerHook.java | 51 +
.../jvm/backtype/storm/hooks/IWorkerHook.java | 44 +
.../storm/logging/ThriftAccessLogger.java | 27 +
.../logging/filters/AccessLoggingFilter.java | 52 +
.../storm/messaging/AddressedTuple.java | 46 +
.../storm/messaging/ConnectionWithStatus.java | 4 +-
.../DeserializingConnectionCallback.java | 60 +
.../backtype/storm/messaging/IConnection.java | 26 +-
.../storm/messaging/IConnectionCallback.java | 31 +
.../jvm/backtype/storm/messaging/IContext.java | 2 +-
.../storm/messaging/TransportFactory.java | 2 +-
.../backtype/storm/messaging/local/Context.java | 164 +
.../backtype/storm/messaging/netty/Client.java | 115 +-
.../backtype/storm/messaging/netty/Context.java | 8 +-
.../storm/messaging/netty/ControlMessage.java | 22 +-
.../messaging/netty/INettySerializable.java | 26 +
.../storm/messaging/netty/ISaslClient.java | 28 +
.../storm/messaging/netty/ISaslServer.java | 26 +
.../backtype/storm/messaging/netty/IServer.java | 26 +
.../netty/KerberosSaslClientHandler.java | 152 +
.../netty/KerberosSaslNettyClient.java | 203 +
.../netty/KerberosSaslNettyClientState.java | 31 +
.../netty/KerberosSaslNettyServer.java | 210 +
.../netty/KerberosSaslNettyServerState.java | 30 +
.../netty/KerberosSaslServerHandler.java | 133 +
.../storm/messaging/netty/MessageBatch.java | 14 +-
.../storm/messaging/netty/MessageDecoder.java | 11 +-
.../netty/NettyRenameThreadFactory.java | 10 +-
.../netty/NettyUncaughtExceptionHandler.java | 35 +
.../storm/messaging/netty/SaslMessageToken.java | 33 +-
.../storm/messaging/netty/SaslNettyClient.java | 28 +-
.../storm/messaging/netty/SaslNettyServer.java | 248 +-
.../messaging/netty/SaslNettyServerState.java | 13 +-
.../messaging/netty/SaslStormClientHandler.java | 41 +-
.../messaging/netty/SaslStormServerHandler.java | 32 +-
.../storm/messaging/netty/SaslUtils.java | 12 +-
.../backtype/storm/messaging/netty/Server.java | 232 +-
.../messaging/netty/StormClientHandler.java | 51 +-
.../netty/StormClientPipelineFactory.java | 11 +-
.../messaging/netty/StormServerHandler.java | 24 +-
.../backtype/storm/metric/EventLoggerBolt.java | 25 +-
.../storm/metric/FileBasedEventLogger.java | 37 +-
.../metric/HttpForwardingMetricsConsumer.java | 80 +
.../metric/HttpForwardingMetricsServer.java | 118 +
.../jvm/backtype/storm/metric/IEventLogger.java | 25 +-
.../storm/metric/LoggingMetricsConsumer.java | 1 -
.../storm/metric/MetricsConsumerBolt.java | 1 -
.../jvm/backtype/storm/metric/SystemBolt.java | 5 -
.../backtype/storm/metric/api/CountMetric.java | 2 -
.../backtype/storm/metric/api/MeanReducer.java | 4 +-
.../storm/metric/api/MultiCountMetric.java | 2 +-
.../storm/metric/api/MultiReducedMetric.java | 2 +-
.../storm/metric/api/rpc/CountShellMetric.java | 3 +-
.../metric/internal/CountStatAndMetric.java | 211 +
.../metric/internal/LatencyStatAndMetric.java | 262 +
.../storm/metric/internal/MetricStatTimer.java | 27 +
.../internal/MultiCountStatAndMetric.java | 112 +
.../internal/MultiLatencyStatAndMetric.java | 109 +
.../storm/metric/internal/RateTracker.java | 165 +
.../AbstractDNSToSwitchMapping.java | 95 +
.../networktopography/DNSToSwitchMapping.java | 50 +
.../DefaultRackDNSToSwitchMapping.java | 52 +
.../backtype/storm/nimbus/ILeaderElector.java | 23 +-
.../nimbus/ITopologyActionNotifierPlugin.java | 43 +
.../jvm/backtype/storm/nimbus/NimbusInfo.java | 29 +-
.../jvm/backtype/storm/scheduler/Cluster.java | 234 +-
.../scheduler/SchedulerAssignmentImpl.java | 15 +-
.../storm/scheduler/SupervisorDetails.java | 63 +-
.../backtype/storm/scheduler/Topologies.java | 27 +-
.../storm/scheduler/TopologyDetails.java | 377 +-
.../backtype/storm/scheduler/WorkerSlot.java | 25 +
.../scheduler/multitenant/DefaultPool.java | 22 +-
.../storm/scheduler/multitenant/FreePool.java | 6 +-
.../scheduler/multitenant/IsolatedPool.java | 32 +-
.../multitenant/MultitenantScheduler.java | 6 +-
.../storm/scheduler/multitenant/Node.java | 17 +-
.../storm/scheduler/multitenant/NodePool.java | 16 +-
.../storm/scheduler/resource/Component.java | 54 +
.../storm/scheduler/resource/RAS_Node.java | 575 +
.../resource/ResourceAwareScheduler.java | 183 +
.../storm/scheduler/resource/ResourceUtils.java | 133 +
.../resource/strategies/IStrategy.java | 37 +
.../strategies/ResourceAwareStrategy.java | 479 +
.../backtype/storm/security/auth/AuthUtils.java | 96 +-
.../auth/DefaultHttpCredentialsPlugin.java | 6 +-
.../security/auth/DefaultPrincipalToLocal.java | 1 -
.../storm/security/auth/IAuthorizer.java | 4 +-
.../security/auth/ICredentialsRenewer.java | 3 +-
.../security/auth/IHttpCredentialsPlugin.java | 2 -
.../storm/security/auth/IPrincipalToLocal.java | 2 +-
.../storm/security/auth/ITransportPlugin.java | 4 -
.../security/auth/KerberosPrincipalToLocal.java | 2 +-
.../storm/security/auth/ReqContext.java | 18 +-
.../security/auth/SaslTransportPlugin.java | 12 +-
.../security/auth/ShellBasedGroupsMapping.java | 10 +-
.../security/auth/SimpleTransportPlugin.java | 8 +-
.../security/auth/SingleUserPrincipal.java | 5 +-
.../storm/security/auth/TBackoffConnect.java | 1 -
.../storm/security/auth/ThriftClient.java | 10 +-
.../storm/security/auth/ThriftServer.java | 6 +-
.../auth/authorizer/DRPCAuthorizerBase.java | 2 +-
.../authorizer/DRPCSimpleACLAuthorizer.java | 19 +-
.../auth/authorizer/DenyAuthorizer.java | 16 +-
.../authorizer/ImpersonationAuthorizer.java | 17 +-
.../auth/authorizer/NoopAuthorizer.java | 12 +-
.../auth/authorizer/SimpleACLAuthorizer.java | 45 +-
.../authorizer/SimpleWhitelistAuthorizer.java | 16 +-
.../auth/digest/ClientCallbackHandler.java | 2 -
.../auth/digest/DigestSaslTransportPlugin.java | 2 -
.../auth/digest/ServerCallbackHandler.java | 5 +-
.../storm/security/auth/kerberos/AutoTGT.java | 10 +-
.../security/auth/kerberos/NoOpTTrasport.java | 20 +-
.../auth/kerberos/ServerCallbackHandler.java | 2 +
.../serialization/BlowfishTupleSerializer.java | 6 +-
.../GzipThriftSerializationDelegate.java | 1 -
.../storm/serialization/ITupleDeserializer.java | 1 -
.../serialization/KryoTupleDeserializer.java | 3 -
.../serialization/KryoValuesDeserializer.java | 3 +-
.../serialization/SerializationFactory.java | 23 +-
.../jvm/backtype/storm/spout/MultiScheme.java | 3 +-
.../backtype/storm/spout/RawMultiScheme.java | 3 +-
.../src/jvm/backtype/storm/spout/RawScheme.java | 9 +-
.../src/jvm/backtype/storm/spout/Scheme.java | 3 +-
.../storm/spout/SchemeAsMultiScheme.java | 3 +-
.../jvm/backtype/storm/spout/ShellSpout.java | 10 +-
.../storm/task/GeneralTopologyContext.java | 15 +-
.../backtype/storm/task/OutputCollector.java | 2 +-
.../src/jvm/backtype/storm/task/ShellBolt.java | 48 +-
.../backtype/storm/task/TopologyContext.java | 9 +-
.../AlternateRackDNSToSwitchMapping.java | 65 +
.../storm/testing/MemoryTransactionalSpout.java | 9 +-
.../testing/OpaqueMemoryTransactionalSpout.java | 8 +-
.../storm/testing/TupleCaptureBolt.java | 4 +-
.../topology/BaseConfigurationDeclarer.java | 31 +-
.../storm/topology/BasicBoltExecutor.java | 2 +-
.../ComponentConfigurationDeclarer.java | 3 +
.../backtype/storm/topology/IWindowedBolt.java | 40 +
.../storm/topology/OutputFieldsGetter.java | 2 +-
.../storm/topology/TopologyBuilder.java | 78 +-
.../storm/topology/WindowedBoltExecutor.java | 224 +
.../storm/topology/base/BaseBatchBolt.java | 1 -
.../topology/base/BaseTransactionalSpout.java | 1 -
.../storm/topology/base/BaseWindowedBolt.java | 179 +
.../TransactionalSpoutBatchExecutor.java | 4 +-
.../TransactionalSpoutCoordinator.java | 2 +-
...uePartitionedTransactionalSpoutExecutor.java | 13 +-
.../PartitionedTransactionalSpoutExecutor.java | 2 +-
.../backtype/storm/tuple/AddressedTuple.java | 48 +
.../src/jvm/backtype/storm/tuple/Fields.java | 10 +-
.../src/jvm/backtype/storm/tuple/MessageId.java | 10 +-
.../src/jvm/backtype/storm/tuple/Tuple.java | 9 +-
.../src/jvm/backtype/storm/tuple/TupleImpl.java | 17 +-
.../jvm/backtype/storm/utils/DRPCClient.java | 1 -
.../backtype/storm/utils/DisruptorQueue.java | 610 +-
.../backtype/storm/utils/InprocMessaging.java | 4 +-
.../storm/utils/KeyedRoundRobinQueue.java | 6 +-
.../jvm/backtype/storm/utils/ListDelegate.java | 6 +-
.../jvm/backtype/storm/utils/LocalState.java | 22 +-
.../src/jvm/backtype/storm/utils/Monitor.java | 3 +-
.../jvm/backtype/storm/utils/MutableObject.java | 6 +-
.../jvm/backtype/storm/utils/NimbusClient.java | 10 +-
.../jvm/backtype/storm/utils/RateTracker.java | 119 -
.../storm/utils/RegisteredGlobalState.java | 6 +-
.../jvm/backtype/storm/utils/RotatingMap.java | 2 +-
.../backtype/storm/utils/ServiceRegistry.java | 2 +-
.../jvm/backtype/storm/utils/ShellProcess.java | 6 +-
.../jvm/backtype/storm/utils/ShellUtils.java | 2 +-
.../StormBoundedExponentialBackoffRetry.java | 3 +-
.../storm/utils/ThriftTopologyUtils.java | 36 +-
.../src/jvm/backtype/storm/utils/Time.java | 16 +-
.../backtype/storm/utils/TransferDrainer.java | 17 +-
.../src/jvm/backtype/storm/utils/Utils.java | 489 +-
.../jvm/backtype/storm/utils/VersionInfo.java | 2 +-
.../storm/validation/ConfigValidation.java | 646 +
.../validation/ConfigValidationAnnotations.java | 214 +
.../storm/validation/ConfigValidationUtils.java | 175 +
.../storm/windowing/CountEvictionPolicy.java | 68 +
.../storm/windowing/CountTriggerPolicy.java | 63 +
.../src/jvm/backtype/storm/windowing/Event.java | 41 +
.../jvm/backtype/storm/windowing/EventImpl.java | 38 +
.../storm/windowing/EvictionPolicy.java | 42 +
.../storm/windowing/TimeEvictionPolicy.java | 52 +
.../storm/windowing/TimeTriggerPolicy.java | 115 +
.../storm/windowing/TriggerHandler.java | 29 +
.../backtype/storm/windowing/TriggerPolicy.java | 42 +
.../backtype/storm/windowing/TupleWindow.java | 26 +
.../storm/windowing/TupleWindowImpl.java | 61 +
.../jvm/backtype/storm/windowing/Window.java | 48 +
.../windowing/WindowLifecycleListener.java | 42 +
.../backtype/storm/windowing/WindowManager.java | 212 +
.../storm/pacemaker/IServerMessageHandler.java | 25 +
.../apache/storm/pacemaker/PacemakerClient.java | 255 +
.../storm/pacemaker/PacemakerClientHandler.java | 75 +
.../apache/storm/pacemaker/PacemakerServer.java | 163 +
.../storm/pacemaker/codec/ThriftDecoder.java | 76 +
.../storm/pacemaker/codec/ThriftEncoder.java | 110 +
.../pacemaker/codec/ThriftNettyClientCodec.java | 94 +
.../pacemaker/codec/ThriftNettyServerCodec.java | 99 +
.../src/jvm/storm/trident/TridentTopology.java | 100 +-
.../trident/drpc/ReturnResultsReducer.java | 4 +-
.../fluent/ChainedAggregatorDeclarer.java | 8 +-
.../jvm/storm/trident/graph/GraphGrouper.java | 22 +-
.../src/jvm/storm/trident/graph/Group.java | 23 +-
.../trident/operation/builtin/SnapshotGet.java | 4 +-
.../operation/builtin/TupleCollectionGet.java | 6 +-
.../storm/trident/partition/GlobalGrouping.java | 5 +-
.../trident/partition/IdentityGrouping.java | 8 +-
.../src/jvm/storm/trident/planner/Node.java | 5 +-
.../storm/trident/planner/PartitionNode.java | 2 -
.../storm/trident/planner/SubtopologyBolt.java | 19 +-
.../processor/MultiReducerProcessor.java | 2 +-
.../jvm/storm/trident/spout/ITridentSpout.java | 51 +-
.../OpaquePartitionedTridentSpoutExecutor.java | 10 +-
.../trident/spout/TridentSpoutExecutor.java | 4 +-
.../trident/topology/TridentBoltExecutor.java | 10 +-
.../topology/TridentTopologyBuilder.java | 23 +-
.../storm/trident/tuple/TridentTupleView.java | 18 +-
.../jvm/storm/trident/util/TridentUtils.java | 33 +-
.../src/native/worker-launcher/impl/main.c | 10 +
.../worker-launcher/impl/worker-launcher.c | 49 +-
.../worker-launcher/impl/worker-launcher.h | 2 +
storm-core/src/py/storm/DistributedRPC-remote | 2 +-
storm-core/src/py/storm/DistributedRPC.py | 20 +-
.../py/storm/DistributedRPCInvocations-remote | 2 +-
.../src/py/storm/DistributedRPCInvocations.py | 41 +-
storm-core/src/py/storm/Nimbus-remote | 51 +-
storm-core/src/py/storm/Nimbus.py | 2383 ++-
storm-core/src/py/storm/constants.py | 2 +-
storm-core/src/py/storm/ttypes.py | 5870 ++++--
storm-core/src/storm.thrift | 262 +-
storm-core/src/ui/public/component.html | 167 +-
storm-core/src/ui/public/css/style.css | 16 +
.../src/ui/public/deep_search_result.html | 155 +
storm-core/src/ui/public/images/bug.png | Bin 0 -> 4045 bytes
storm-core/src/ui/public/images/search.png | Bin 0 -> 2354 bytes
storm-core/src/ui/public/images/statistic.png | Bin 0 -> 488 bytes
storm-core/src/ui/public/index.html | 10 +-
storm-core/src/ui/public/js/script.js | 20 +
.../src/ui/public/js/typeahead.jquery.min.js | 7 +
storm-core/src/ui/public/js/visualization.js | 92 +-
storm-core/src/ui/public/logviewer_search.html | 65 +
storm-core/src/ui/public/search_result.html | 100 +
.../templates/component-page-template.html | 55 +-
.../deep-search-result-page-template.html | 66 +
.../public/templates/index-page-template.html | 56 +-
.../logviewer-search-page-template.html | 44 +
.../templates/search-result-page-template.html | 60 +
.../templates/topology-page-template.html | 197 +-
.../src/ui/public/templates/user-template.html | 27 +-
storm-core/src/ui/public/topology.html | 168 +-
.../test/clj/backtype/storm/cluster_test.clj | 15 +-
.../test/clj/backtype/storm/config_test.clj | 186 -
.../test/clj/backtype/storm/grouping_test.clj | 90 +-
.../clj/backtype/storm/integration_test.clj | 12 +-
.../test/clj/backtype/storm/logviewer_test.clj | 730 +-
.../storm/messaging/netty_integration_test.clj | 3 +-
.../storm/messaging/netty_unit_test.clj | 288 +-
.../test/clj/backtype/storm/messaging_test.clj | 28 +-
.../test/clj/backtype/storm/metrics_test.clj | 2 +-
.../test/clj/backtype/storm/multilang_test.clj | 4 +-
.../test/clj/backtype/storm/nimbus_test.clj | 199 +-
.../scheduler/multitenant_scheduler_test.clj | 34 +-
.../scheduler/resource_aware_scheduler_test.clj | 669 +
.../test/clj/backtype/storm/scheduler_test.clj | 3 +-
.../auth/DefaultHttpCredentialsPlugin_test.clj | 40 +-
.../clj/backtype/storm/serialization_test.clj | 14 +-
.../test/clj/backtype/storm/supervisor_test.clj | 397 +-
.../test/clj/backtype/storm/testing4j_test.clj | 1 +
.../clj/backtype/storm/transactional_test.clj | 5 +-
.../test/clj/backtype/storm/worker_test.clj | 179 +-
.../storm/pacemaker_state_factory_test.clj | 150 +
.../clj/org/apache/storm/pacemaker_test.clj | 242 +
.../jvm/backtype/storm/TestConfigValidate.java | 660 +
.../metric/internal/CountStatAndMetricTest.java | 86 +
.../internal/LatencyStatAndMetricTest.java | 83 +
.../storm/metric/internal/RateTrackerTest.java | 94 +
.../nimbus/InMemoryTopologyActionNotifier.java | 53 +
.../storm/topology/TopologyBuilderTest.java | 5 +
.../utils/DisruptorQueueBackpressureTest.java | 17 +-
.../storm/utils/DisruptorQueueTest.java | 106 +-
.../backtype/storm/utils/MockTupleHelpers.java | 40 +
.../backtype/storm/utils/RateTrackerTest.java | 62 -
.../storm/utils/ThriftTopologyUtilsTest.java | 94 +
.../storm/windowing/WindowManagerTest.java | 250 +
.../jvm/storm/trident/TestTridentTopology.java | 76 +
storm-dist/binary/LICENSE | 29 +
storm-dist/binary/src/main/assembly/binary.xml | 37 -
567 files changed, 70259 insertions(+), 14812 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/31b49594/CHANGELOG.md
----------------------------------------------------------------------
diff --cc CHANGELOG.md
index c5d052b,4eb137b..f513a7f
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@@ -1,6 -1,99 +1,101 @@@
## 0.11.0
+ * STORM-1060: Serialize Calcite plans into JSON format.
+ * STORM-1062: Establish the basic structure of the code generator.
+ * STORM-1341: Let topology have own heartbeat timeout for multilang subprocess
+ * STORM-1207: Added flux support for IWindowedBolt
+ * STORM-1352: Trident should support writing to multiple Kafka clusters.
+ * STORM-1220: Avoid double copying in the Kafka spout.
+ * STORM-1340: Use Travis-CI build matrix to improve test execution times
+ * STORM-1126: Allow a configMethod that takes no arguments (Flux)
+ * STORM-1203: worker metadata file creation doesn't use storm.log.dir config
+ * STORM-1349: [Flux] Allow constructorArgs to take Maps as arguments
+ * STORM-126: Add Lifecycle support API for worker nodes
+ * STORM-1213: Remove sigar binaries from source tree
+ * STORM-885: Heartbeat Server (Pacemaker)
+ * STORM-1221: Create a common interface for all Trident spout.
+ * STORM-1198: Web UI to show resource usages and Total Resources on all supervisors
+ * STORM-1167: Add windowing support for storm core.
+ * STORM-1215: Use Async Loggers to avoid locking and logging overhead
+ * STORM-1204: Logviewer should graceful report page-not-found instead of 500 for bad topo-id etc
+ * STORM-831: Add BugTracker and Central Logging URL to UI
+ * STORM-1208: UI: NPE seen when aggregating bolt streams stats
+ * STORM-1016: Generate trident bolt ids with sorted group names
+ * STORM-1190: System Load too high after recent changes
+ * STORM-1098: Nimbus hook for topology actions.
+ * STORM-1145: Have IConnection push tuples instead of pull them
+ * STORM-1191: bump timeout by 50% due to intermittent travis build failures
+ * STORM-794: Modify Spout async loop to treat activate/deactivate ASAP
+ * STORM-1196: Upgrade to thrift 0.9.3
+ * STORM-1155: Supervisor recurring health checks
+ * STORM-1189: Maintain wire compatability with 0.10.x versions of storm.
+ * STORM-1185: replace nimbus.host with nimbus.seeds
+ * STORM-1164: Code cleanup for typos, warnings and conciseness.
+ * STORM-902: Simple Log Search.
+ * STORM-1052: TridentKafkaState uses new Kafka Producer API.
+ * STORM-1182: Removing and wrapping some exceptions in ConfigValidation to make code cleaner
+ * STORM-1134. Windows: Fix log4j config.
+ * STORM-1127: allow for boolean arguments (Flux)
+ * STORM-1180: FLUX logo wasn't appearing quite right
+ * STORM-1138: Storm-hdfs README should be updated with Avro Bolt information
+ * STORM-1154: SequenceFileBolt needs unit tests
+ * STORM-162: Load Aware Shuffle Grouping
+ * STORM-1158: Storm metrics to profile various storm functions
+ * STORM-1161: Add License headers and add rat checks to builds
+ * STORM-1165: normalize the scales of CPU/Mem/Net when choosing the best node for Resource Aware Scheduler
+ * STORM-1163: use rmr rather than rmpath for remove worker-root
+ * STORM-1170: Fix the producer alive issue in DisruptorQueueTest
+ * STORM-1168: removes noisy log message & a TODO
+ * STORM-1143: Validate topology Configs during topology submission
+ * STORM-1157: Adding dynamic profiling for worker, restarting worker, jstack, heap dump, and profiling
+ * STORM-1123: TupleImpl - Unnecessary variable initialization.
+ * STORM-1153: Use static final instead of just static for class members.
+ * STORM-817: Kafka Wildcard Topic Support.
+ * STORM-40: Turn worker garbage collection and heapdump on by default.
+ * STORM-1152: Change map keySet iteration to entrySet iteration for efficiency.
+ * STORM-1147: Storm JDBCBolt should add validation to ensure either insertQuery or table name is specified and not both.
+ * STORM-1151: Batching in DisruptorQueue
+ * STORM-350: Update disruptor to latest version (3.3.2)
+ * STORM-697: Support for Emitting Kafka Message Offset and Partition
+ * STORM-1074: Add Avro HDFS bolt
+ * STORM-566: Improve documentation including incorrect Kryo ser. framework docs
+ * STORM-1073: Refactor AbstractHdfsBolt
+ * STORM-1128: Make metrics fast
+ * STORM-1122: Fix the format issue in Utils.java
+ * STORM-1111: Fix Validation for lots of different configs
+ * STORM-1125: Adding separate ZK client for read in Nimbus ZK State
+ * STORM-1121: Remove method call to avoid overhead during topology submission time
+ * STORM-1120: Fix keyword (schema -> scheme) from main-routes
+ * STORM-1115: Stale leader-lock key effectively bans all nodes from becoming leaders
+ * STORM-1119: Create access logging for all daemons
+ * STORM-1117: Adds visualization-init route previously missing
+ * STORM-1118: Added test to compare latency vs. throughput in storm.
+ * STORM-1110: Fix Component Page for system components
+ * STORM-1093: Launching Workers with resources specified in resource-aware schedulers
+ * STORM-1102: Add a default flush interval for HiveBolt
+ * STORM-1112: Add executor id to the thread name of the executor thread for debug
+ * STORM-1079: Batch Puts to HBase
+ * STORM-1084: Improve Storm config validation process to use java annotations instead of *_SCHEMA format
+ * STORM-1106: Netty should not limit attempts to reconnect
+ * STORM-1103: Changes log message to DEBUG from INFO
+ * STORM-1104: Nimbus HA fails to find newly downloaded code files
+ * STORM-1087: Avoid issues with transfer-queue backpressure.
+ * STORM-893: Resource Aware Scheduling (Experimental)
+ * STORM-1095: Tuple.getSourceGlobalStreamid() has wrong camel-case naming
+ * STORM-1091: Add unit test for tick tuples to HiveBolt and HdfsBolt
+ * STORM-1090: Nimbus HA should support `storm.local.hostname`
+ * STORM-820: Aggregate topo stats on nimbus, not ui
+ * STORM-412: Allow users to modify logging levels of running topologies
+ * STORM-1078: Updated RateTracker to be thread safe
+ * STORM-1082: fix nits for properties in kafka tests
+ * STORM-993: include uptimeSeconds as JSON integer field
+ * STORM-1053: Update storm-kafka README for new producer API confs.
+ * STORM-1058: create CLI kill_workers to kill workers on a supervisor node
+ * STORM-1063: support relative log4j conf dir for both daemons and workers
+ * STORM-1059: Upgrade Storm to use Clojure 1.7.0
+ * STORM-1069: add check case for external change of 'now' value.
+ * STORM-969: HDFS Bolt can end up in an unrecoverable state.
+ * STORM-1068: Configure request.required.acks to be 1 in KafkaUtilsTest for sync
+ * STORM-1017: If ignoreZkOffsets set true,KafkaSpout will reset zk offset when recover from failure.
* STORM-1054: Excessive logging ShellBasedGroupsMapping if the user doesn't have any groups.
* STORM-954: Toplogy Event Inspector
* STORM-862: Pluggable System Metrics
http://git-wip-us.apache.org/repos/asf/storm/blob/31b49594/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/TestCompilerUtils.java
----------------------------------------------------------------------
diff --cc external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/TestCompilerUtils.java
index 0e5fa0b,0000000..092230f
mode 100644,000000..100644
--- a/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/TestCompilerUtils.java
+++ b/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/TestCompilerUtils.java
@@@ -1,48 -1,0 +1,65 @@@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ * <p>
++ * http://www.apache.org/licenses/LICENSE-2.0
++ * <p>
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
+package org.apache.storm.sql.compiler;
+
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.StreamableTable;
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.tools.*;
+
+public class TestCompilerUtils {
+ public static CalciteState sqlOverDummyTable(String sql)
+ throws RelConversionException, ValidationException, SqlParseException {
+ SchemaPlus schema = Frameworks.createRootSchema(true);
+ JavaTypeFactory typeFactory = new JavaTypeFactoryImpl
+ (RelDataTypeSystem.DEFAULT);
+ StreamableTable streamableTable = new CompilerUtil.TableBuilderInfo(typeFactory)
+ .field("ID", SqlTypeName.INTEGER).build();
+ Table table = streamableTable.stream();
+ schema.add("FOO", table);
+ schema.add("BAR", table);
+ FrameworkConfig config = Frameworks.newConfigBuilder().defaultSchema(
+ schema).build();
+ Planner planner = Frameworks.getPlanner(config);
+ SqlNode parse = planner.parse(sql);
+ SqlNode validate = planner.validate(parse);
+ RelNode tree = planner.convert(validate);
+ return new CalciteState(schema, tree);
+ }
+
+ public static class CalciteState {
+ final SchemaPlus schema;
+ final RelNode tree;
+
+ private CalciteState(SchemaPlus schema, RelNode tree) {
+ this.schema = schema;
+ this.tree = tree;
+ }
+
+ public SchemaPlus schema() { return schema; }
+ public RelNode tree() { return tree; }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/31b49594/external/sql/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonScheme.java
----------------------------------------------------------------------
diff --cc external/sql/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonScheme.java
index 80037c6,0000000..1b45b30
mode 100644,000000..100644
--- a/external/sql/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonScheme.java
+++ b/external/sql/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonScheme.java
@@@ -1,56 -1,0 +1,58 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.sql.kafka;
+
+import backtype.storm.spout.Scheme;
+import backtype.storm.tuple.Fields;
++import backtype.storm.utils.Utils;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.IOException;
++import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+public class JsonScheme implements Scheme {
+ private final List<String> fields;
+
+ JsonScheme(List<String> fields) {
+ this.fields = fields;
+ }
+
+ @Override
- public List<Object> deserialize(byte[] ser) {
++ public List<Object> deserialize(ByteBuffer ser) {
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ @SuppressWarnings("unchecked")
- HashMap<String, Object> map = mapper.readValue(ser, HashMap.class);
++ HashMap<String, Object> map = mapper.readValue(Utils.toByteArray(ser), HashMap.class);
+ ArrayList<Object> list = new ArrayList<>();
+ for (String f : fields) {
+ list.add(map.get(f));
+ }
+ return list;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Fields getOutputFields() {
+ return new Fields(fields);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/31b49594/external/sql/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestJsonRepresentation.java
----------------------------------------------------------------------
diff --cc external/sql/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestJsonRepresentation.java
index d2898e8,0000000..5973672
mode 100644,000000..100644
--- a/external/sql/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestJsonRepresentation.java
+++ b/external/sql/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestJsonRepresentation.java
@@@ -1,50 -1,0 +1,50 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.sql.kafka;
+
+import backtype.storm.utils.Utils;
+import com.google.common.collect.Lists;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class TestJsonRepresentation {
+ @Test
+ public void testJsonScheme() {
+ final List<String> fields = Lists.newArrayList("ID", "val");
+ final String s = "{\"ID\": 1, \"val\": \"2\"}";
+ JsonScheme scheme = new JsonScheme(fields);
- List<Object> o = scheme.deserialize(s.getBytes(Charset.defaultCharset()));
++ List<Object> o = scheme.deserialize(ByteBuffer.wrap(s.getBytes(Charset.defaultCharset())));
+ assertArrayEquals(new Object[] {1, "2"}, o.toArray());
+ }
+
+ @Test
+ public void testJsonSerializer() {
+ final List<String> fields = Lists.newArrayList("ID", "val");
+ List<Object> o = Lists.<Object> newArrayList(1, "2");
+ JsonSerializer s = new JsonSerializer(fields);
+ ByteBuffer buf = s.write(o, null);
+ byte[] b = Utils.toByteArray(buf);
+ assertEquals("{\"ID\":1,\"val\":\"2\"}", new String(b));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/31b49594/external/sql/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java
----------------------------------------------------------------------
diff --cc external/sql/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java
index 531f764,0000000..418bc68
mode 100644,000000..100644
--- a/external/sql/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java
+++ b/external/sql/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java
@@@ -1,103 -1,0 +1,107 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.sql.kafka;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import kafka.javaapi.producer.Producer;
+import kafka.producer.KeyedMessage;
++import org.apache.kafka.clients.producer.KafkaProducer;
++import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.storm.sql.kafka.KafkaDataSourcesProvider.KafkaTridentSink;
+import org.apache.storm.sql.runtime.DataSourcesRegistry;
+import org.apache.storm.sql.runtime.FieldInfo;
+import org.apache.storm.sql.runtime.ISqlTridentDataSource;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+import org.mockito.internal.util.reflection.Whitebox;
+import storm.kafka.trident.TridentKafkaState;
+import storm.trident.tuple.TridentTuple;
+
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
++import java.util.concurrent.Future;
+
+import static org.mockito.Mockito.*;
+
+public class TestKafkaDataSourcesProvider {
+ private static final List<FieldInfo> FIELDS = ImmutableList.of(
+ new FieldInfo("ID", int.class, true),
+ new FieldInfo("val", String.class, false));
+ private static final List<String> FIELD_NAMES = ImmutableList.of("ID", "val");
+ private static final JsonSerializer SERIALIZER = new JsonSerializer(FIELD_NAMES);
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testKafkaSink() {
+ ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(
+ URI.create("kafka://mock?topic=foo"), null, null, FIELDS);
+ Assert.assertNotNull(ds);
+ KafkaTridentSink sink = (KafkaTridentSink) ds.getConsumer();
+ sink.prepare(null, null);
+ TridentKafkaState state = (TridentKafkaState) Whitebox.getInternalState(sink, "state");
- Producer producer = mock(Producer.class);
++ KafkaProducer producer = mock(KafkaProducer.class);
++ doReturn(mock(Future.class)).when(producer).send(any(ProducerRecord.class));
+ Whitebox.setInternalState(state, "producer", producer);
+ List<TridentTuple> tupleList = mockTupleList();
+ for (TridentTuple t : tupleList) {
+ state.updateState(Collections.singletonList(t), null);
+ verify(producer).send(argThat(new KafkaMessageMatcher(t)));
+ }
+ verifyNoMoreInteractions(producer);
+ }
+
+ private static List<TridentTuple> mockTupleList() {
+ List<TridentTuple> tupleList = new ArrayList<>();
+ TridentTuple t0 = mock(TridentTuple.class);
+ TridentTuple t1 = mock(TridentTuple.class);
+ doReturn(1).when(t0).get(0);
+ doReturn(2).when(t1).get(0);
+ doReturn(Lists.<Object>newArrayList(1, "2")).when(t0).getValues();
+ doReturn(Lists.<Object>newArrayList(2, "3")).when(t1).getValues();
+ tupleList.add(t0);
+ tupleList.add(t1);
+ return tupleList;
+ }
+
- private static class KafkaMessageMatcher extends ArgumentMatcher<KeyedMessage> {
++ private static class KafkaMessageMatcher extends ArgumentMatcher<ProducerRecord> {
+ private static final int PRIMARY_INDEX = 0;
+ private final TridentTuple tuple;
+
+ private KafkaMessageMatcher(TridentTuple tuple) {
+ this.tuple = tuple;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public boolean matches(Object o) {
- KeyedMessage<Object, ByteBuffer> m = (KeyedMessage<Object,ByteBuffer>)o;
++ ProducerRecord<Object, ByteBuffer> m = (ProducerRecord<Object,ByteBuffer>)o;
+ if (m.key() != tuple.get(PRIMARY_INDEX)) {
+ return false;
+ }
- ByteBuffer buf = m.message();
++ ByteBuffer buf = m.value();
+ ByteBuffer b = SERIALIZER.write(tuple.getValues(), null);
+ return b.equals(buf);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/31b49594/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index faf94ab,0da88b0..3ed8e3f
--- a/pom.xml
+++ b/pom.xml
@@@ -225,10 -210,37 +210,39 @@@
<hive.version>0.14.0</hive.version>
<hadoop.version>2.6.0</hadoop.version>
<kryo.version>2.21</kryo.version>
+ <servlet.version>2.5</servlet.version>
+ <joda-time.version>2.3</joda-time.version>
+ <jackson.version>2.3.1</jackson.version>
+ <thrift.version>0.9.3</thrift.version>
+ <junit.version>4.11</junit.version>
+ <metrics-clojure.version>2.5.1</metrics-clojure.version>
+ <hdrhistogram.version>2.1.7</hdrhistogram.version>
+ <calcite.version>1.4.0-incubating</calcite.version>
- <jackson.version>2.6.3</jackson.version>
</properties>
+ <modules>
+ <module>storm-multilang/javascript</module>
+ <module>storm-multilang/python</module>
+ <module>storm-multilang/ruby</module>
+ <module>storm-buildtools/maven-shade-clojure-transformer</module>
+ <module>storm-buildtools/storm-maven-plugins</module>
+ <module>storm-core</module>
+ <module>external/storm-kafka</module>
+ <module>external/storm-hdfs</module>
+ <module>external/storm-hbase</module>
+ <module>external/storm-hive</module>
+ <module>external/storm-jdbc</module>
+ <module>external/storm-redis</module>
+ <module>external/storm-eventhubs</module>
+ <module>external/flux</module>
+ <module>external/storm-elasticsearch</module>
+ <module>external/storm-solr</module>
+ <module>external/storm-metrics</module>
++ <module>external/sql</module>
+ <module>examples/storm-starter</module>
+ </modules>
+
+
<profiles>
<profile>
<id>sign</id>
@@@ -577,22 -660,11 +662,21 @@@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-core</artifactId>
- <version>${calcite.version}</version>
++ <groupId>org.apache.calcite</groupId>
++ <artifactId>calcite-core</artifactId>
++ <version>${calcite.version}</version>
++ </dependency>
++ <dependency>
++ <groupId>com.fasterxml.jackson.core</groupId>
++ <artifactId>jackson-databind</artifactId>
++ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson.version}</version>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
</dependency>
- <!-- used by examples/storm-starter -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- <scope>test</scope>
- </dependency>
</dependencies>
</dependencyManagement>
@@@ -724,6 -796,73 +808,76 @@@
<artifactId>clojure-maven-plugin</artifactId>
<extensions>true</extensions>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>0.11</version>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludeSubProjects>false</excludeSubProjects>
+ <excludes>
+ <!-- exclude maven artifacts -->
+ <exclude>**/target/**</exclude>
+ <!-- exclude intellij projects -->
+ <exclude>**/*.iml</exclude>
+ <exclude>**/.idea/**</exclude>
+ <!-- module specific testing artifacts -->
+ <exclude>**/metastore_db/**</exclude>
+
+ <!-- exclude CHANGELOG, VERSION, AND TODO files -->
+ <exclude>**/CHANGELOG.md</exclude>
+ <exclude>**/README.md</exclude>
+ <exclude>**/README.markdown</exclude>
+ <exclude>**/DEVELOPER.md</exclude>
+ <exclude>**/BYLAWS.md</exclude>
+ <exclude>**/STORM-UI-REST-API.md</exclude>
+ <exclude>SECURITY.md</exclude>
+ <exclude>VERSION</exclude>
+ <exclude>TODO</exclude>
+ <!-- thrift-generated code -->
+ <exclude>**/src/py/**</exclude>
+
+ <!-- the following are in the LICENSE file -->
+ <exclude>**/src/ui/public/js/jquery.dataTables.1.10.4.min.js</exclude>
+ <exclude>**/src/ui/public/css/jquery.dataTables.1.10.4.min.css</exclude>
+ <exclude>**/src/ui/public/images/*</exclude>
+ <exclude>**/src/ui/public/js/bootstrap-3.3.1.min.js</exclude>
+ <exclude>**/src/ui/public/css/bootstrap-3.3.1.min.css</exclude>
+ <exclude>**/src/ui/public/js/dataTables.bootstrap.min.js</exclude>
+ <exclude>**/src/ui/public/css/dataTables.bootstrap.css</exclude>
+ <exclude>**/src/ui/public/js/jsonFormatter.min.js</exclude>
+ <exclude>**/src/ui/public/css/jsonFormatter.min.css</exclude>
+ <exclude>**/src/ui/public/js/jquery-1.11.1.min.js</exclude>
+ <exclude>**/src/ui/public/js/jquery.cookies.2.2.0.min.js</exclude>
+ <exclude>**/src/ui/public/js/moment.min.js</exclude>
+ <exclude>**/src/ui/public/js/jquery.blockUI.min.js</exclude>
+ <exclude>**/src/ui/public/js/url.min.js</exclude>
+ <exclude>**/src/ui/public/js/arbor.js</exclude>
+ <exclude>**/src/ui/public/js/arbor-graphics.js</exclude>
+ <exclude>**/src/ui/public/js/arbor-tween.js</exclude>
+ <exclude>**/src/ui/public/js/jquery.mustache.js</exclude>
+ <exclude>**/src/ui/public/js/typeahead.jquery.min.js</exclude>
+
+ <!-- generated by shade plugin -->
+ <exclude>**/dependency-reduced-pom.xml</exclude>
+
+ <exclude>**/docs/**</exclude>
+ <exclude>**/.git/**</exclude>
+ <exclude>**/derby.log</exclude>
+ <exclude>**/src/dev/**</exclude>
++ <!-- Storm SQL -->
++ <exclude>**/src/codegen/config.fmpp</exclude>
++ <exclude>**/src/codegen/data/Parser.tdd</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/storm/blob/31b49594/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/utils/Utils.java
index b80aa11,c086be2..47d2332
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@@ -57,9 -83,10 +85,11 @@@ import org.apache.thrift.TSerializer
public class Utils {
private static final Logger LOG = LoggerFactory.getLogger(Utils.class);
public static final String DEFAULT_STREAM_ID = "default";
+ private static ThreadLocal<TSerializer> threadSer = new ThreadLocal<TSerializer>();
+ private static ThreadLocal<TDeserializer> threadDes = new ThreadLocal<TDeserializer>();
private static SerializationDelegate serializationDelegate;
+ private static ClassLoader cl = ClassLoader.getSystemClassLoader();
static {
Map conf = readStormConfig();
@@@ -659,30 -744,123 +747,133 @@@
return delegate;
}
- public static void handleUncaughtException(Throwable t) {
- if (t != null && t instanceof Error) {
- if (t instanceof OutOfMemoryError) {
+ public static void handleUncaughtException(Throwable t) {
+ if (t != null && t instanceof Error) {
+ if (t instanceof OutOfMemoryError) {
+ try {
+ System.err.println("Halting due to Out Of Memory Error..." + Thread.currentThread().getName());
+ } catch (Throwable err) {
+ //Again we don't want to exit because of logging issues.
+ }
+ Runtime.getRuntime().halt(-1);
+ } else {
+ //Running in daemon mode, we would pass Error to calling thread.
+ throw (Error) t;
+ }
+ }
+ }
+
+ /**
+ * Given a File input it will unzip the file in a the unzip directory
+ * passed as the second parameter
+ * @param inFile The zip file as input
+ * @param unzipDir The unzip directory where to unzip the zip file.
+ * @throws IOException
+ */
+ public static void unZip(File inFile, File unzipDir) throws IOException {
+ Enumeration<? extends ZipEntry> entries;
+ ZipFile zipFile = new ZipFile(inFile);
+
try {
- System.err.println("Halting due to Out Of Memory Error..." + Thread.currentThread().getName());
- } catch (Throwable err) {
- //Again we don't want to exit because of logging issues.
- }
- Runtime.getRuntime().halt(-1);
- } else {
- //Running in daemon mode, we would pass Error to calling thread.
- throw (Error) t;
- }
- }
- }
-
- @VisibleForTesting
- public static void setClassLoaderForJavaDeSerialize(ClassLoader cl) {
- Utils.cl = cl;
- }
-
- @VisibleForTesting
- public static void resetClassLoaderForJavaDeSerialize() {
- Utils.cl = ClassLoader.getSystemClassLoader();
- }
+ entries = zipFile.entries();
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ if (!entry.isDirectory()) {
+ InputStream in = zipFile.getInputStream(entry);
+ try {
+ File file = new File(unzipDir, entry.getName());
+ if (!file.getParentFile().mkdirs()) {
+ if (!file.getParentFile().isDirectory()) {
+ throw new IOException("Mkdirs failed to create " +
+ file.getParentFile().toString());
+ }
+ }
+ OutputStream out = new FileOutputStream(file);
+ try {
+ byte[] buffer = new byte[8192];
+ int i;
+ while ((i = in.read(buffer)) != -1) {
+ out.write(buffer, 0, i);
+ }
+ } finally {
+ out.close();
+ }
+ } finally {
+ in.close();
+ }
+ }
+ }
+ } finally {
+ zipFile.close();
+ }
+ }
+
+ /**
+ * Given a zip File input it will return its size
+ * Only works for zip files whose uncompressed size is less than 4 GB,
+ * otherwise returns the size module 2^32, per gzip specifications
+ * @param myFile The zip file as input
+ * @throws IOException
+ * @return zip file size as a long
+ */
+ public static long zipFileSize(File myFile) throws IOException{
+ RandomAccessFile raf = new RandomAccessFile(myFile, "r");
+ raf.seek(raf.length() - 4);
+ long b4 = raf.read();
+ long b3 = raf.read();
+ long b2 = raf.read();
+ long b1 = raf.read();
+ long val = (b1 << 24) | (b2 << 16) + (b3 << 8) + b4;
+ raf.close();
+ return val;
+ }
+
+ public static double zeroIfNaNOrInf(double x) {
+ return (Double.isNaN(x) || Double.isInfinite(x)) ? 0.0 : x;
+ }
+
+ /**
+ * parses the arguments to extract jvm heap memory size in MB.
+ * @param input
+ * @param defaultValue
+ * @return the value of the JVM heap memory setting (in MB) in a java command.
+ */
+ public static Double parseJvmHeapMemByChildOpts(String input, Double defaultValue) {
+ if (input != null) {
+ Pattern optsPattern = Pattern.compile("Xmx[0-9]+[mkgMKG]");
+ Matcher m = optsPattern.matcher(input);
+ String memoryOpts = null;
+ while (m.find()) {
+ memoryOpts = m.group();
+ }
+ if (memoryOpts != null) {
+ int unit = 1;
+ if (memoryOpts.toLowerCase().endsWith("k")) {
+ unit = 1024;
+ } else if (memoryOpts.toLowerCase().endsWith("m")) {
+ unit = 1024 * 1024;
+ } else if (memoryOpts.toLowerCase().endsWith("g")) {
+ unit = 1024 * 1024 * 1024;
+ }
+ memoryOpts = memoryOpts.replaceAll("[a-zA-Z]", "");
+ Double result = Double.parseDouble(memoryOpts) * unit / 1024.0 / 1024.0;
+ return (result < 1.0) ? 1.0 : result;
+ } else {
+ return defaultValue;
+ }
+ } else {
+ return defaultValue;
+ }
+ }
++
++ @VisibleForTesting
++ public static void setClassLoaderForJavaDeSerialize(ClassLoader cl) {
++ Utils.cl = cl;
++ }
++
++ @VisibleForTesting
++ public static void resetClassLoaderForJavaDeSerialize() {
++ Utils.cl = ClassLoader.getSystemClassLoader();
++ }
}