You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by bo...@apache.org on 2018/09/18 23:01:08 UTC
[11/47] samza git commit: Merge branch 'master' of
https://github.com/apache/samza
Merge branch 'master' of https://github.com/apache/samza
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/0edf343b
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/0edf343b
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/0edf343b
Branch: refs/heads/NewKafkaSystemConsumer
Commit: 0edf343b8c5360437fa10cc5a543696a4edcc319
Parents: 67e611e f249e71
Author: Boris S <bo...@apache.org>
Authored: Fri Jun 8 10:15:36 2018 -0700
Committer: Boris S <bo...@apache.org>
Committed: Fri Jun 8 10:15:36 2018 -0700
----------------------------------------------------------------------
.travis.yml | 3 +-
README.md | 4 +-
bin/check-all.sh | 2 +-
bin/integration-tests.sh | 3 +-
build.gradle | 24 +-
docs/Gemfile | 7 +-
docs/Gemfile.lock | 95 ++-
docs/README.md | 4 +-
docs/community/committers.md | 8 +-
.../documentation/versioned/azure/eventhubs.md | 6 +-
.../versioned/container/metrics-table.html | 8 +
.../versioned/jobs/configuration-table.html | 91 ++-
.../documentation/versioned/jobs/logging.md | 2 +-
.../documentation/versioned/rest/monitors.md | 8 +
.../versioned/deploy-samza-job-from-hdfs.md | 2 +-
.../tutorials/versioned/deploy-samza-to-CDH.md | 4 +-
.../versioned/hello-samza-high-level-yarn.md | 2 +-
.../versioned/hello-samza-high-level-zk.md | 2 +-
.../versioned/remote-debugging-samza.md | 2 +-
.../versioned/run-in-multi-node-yarn.md | 4 +-
.../versioned/samza-rest-getting-started.md | 2 +-
docs/startup/download/index.md | 19 +-
docs/startup/hello-samza/versioned/index.md | 2 +-
gradle.properties | 2 +-
gradle/buildscript.gradle | 2 +-
gradle/dependency-versions-scala-2.10.gradle | 2 +-
gradle/dependency-versions-scala-2.11.gradle | 2 +-
gradle/dependency-versions-scala-2.12.gradle | 2 +-
gradle/dependency-versions.gradle | 7 +-
gradle/release.gradle | 11 +-
.../samza/application/StreamApplication.java | 7 +-
.../samza/checkpoint/CheckpointManager.java | 12 +-
.../java/org/apache/samza/config/MapConfig.java | 9 +-
.../apache/samza/operators/MessageStream.java | 34 +-
.../apache/samza/operators/TimerRegistry.java | 41 ++
.../operators/functions/ClosableFunction.java | 3 +
.../operators/functions/FilterFunction.java | 3 +-
.../operators/functions/FlatMapFunction.java | 3 +-
.../operators/functions/FoldLeftFunction.java | 16 +-
.../operators/functions/InitableFunction.java | 3 +-
.../samza/operators/functions/JoinFunction.java | 3 +-
.../samza/operators/functions/MapFunction.java | 3 +-
.../samza/operators/functions/SinkFunction.java | 3 +-
.../functions/StreamTableJoinFunction.java | 3 +-
.../operators/functions/SupplierFunction.java | 38 ++
.../operators/functions/TimerFunction.java | 65 ++
.../samza/operators/triggers/AnyTrigger.java | 10 +-
.../samza/operators/triggers/Trigger.java | 3 +-
.../apache/samza/operators/windows/Window.java | 3 +-
.../apache/samza/operators/windows/Windows.java | 53 +-
.../windows/internal/WindowInternal.java | 32 +-
.../apache/samza/runtime/ApplicationRunner.java | 19 +
.../samza/serializers/SerializableSerde.java | 2 +-
.../org/apache/samza/sql/udfs/ScalarUdf.java | 4 +-
.../samza/storage/kv/KeyValueSnapshot.java | 42 ++
.../apache/samza/storage/kv/KeyValueStore.java | 14 +
.../org/apache/samza/system/StreamSpec.java | 39 +-
.../org/apache/samza/system/SystemAdmin.java | 23 +-
.../samza/system/SystemStreamPartition.java | 4 +-
.../table/LocalStoreBackedTableProvider.java | 37 --
.../org/apache/samza/table/ReadWriteTable.java | 9 +-
.../org/apache/samza/table/ReadableTable.java | 11 +-
.../org/apache/samza/table/TableProvider.java | 18 +-
.../java/org/apache/samza/table/TableSpec.java | 12 +-
.../java/org/apache/samza/task/TaskContext.java | 19 +
.../org/apache/samza/task/TimerCallback.java | 34 ++
.../java/org/apache/samza/util/RateLimiter.java | 108 ++++
.../samza/operators/windows/TestWindowPane.java | 2 +-
.../autoscaling/deployer/ConfigManager.java | 4 +-
.../samza/system/kinesis/KinesisConfig.java | 28 +-
.../samza/coordinator/AzureJobCoordinator.java | 93 +--
.../samza/coordinator/data/ProcessorEntity.java | 8 +
.../scheduler/JMVersionUpgradeScheduler.java | 9 +-
.../eventhub/EventHubClientManagerFactory.java | 3 +-
.../samza/system/eventhub/EventHubConfig.java | 117 +++-
.../eventhub/SamzaEventHubClientManager.java | 39 +-
.../eventhub/admin/EventHubSystemAdmin.java | 138 +++--
.../consumer/EventHubSystemConsumer.java | 304 ++++++----
.../system/eventhub/metrics/SamzaHistogram.java | 69 ++-
.../eventhub/producer/AsyncSystemProducer.java | 218 +++++++
.../producer/EventHubSystemProducer.java | 378 +++++++-----
.../java/org/apache/samza/util/TableUtils.java | 49 +-
.../samza/system/eventhub/MockEventData.java | 21 +-
.../MockEventHubClientManagerFactory.java | 25 +-
.../eventhub/MockEventHubConfigFactory.java | 18 +-
.../system/eventhub/TestMetricsRegistry.java | 7 +-
.../consumer/TestEventHubSystemConsumer.java | 75 ++-
.../producer/ITestEventHubSystemProducer.java | 10 +-
.../producer/TestEventHubSystemProducer.java | 232 +++++--
samza-azure/src/test/resources/log4j.xml | 43 ++
.../AbstractContainerAllocator.java | 2 +-
.../ClusterBasedJobCoordinator.java | 78 ++-
.../clustermanager/ContainerProcessManager.java | 14 +-
.../HostAwareContainerAllocator.java | 17 +-
.../clustermanager/ResourceRequestState.java | 53 +-
.../apache/samza/config/JavaStorageConfig.java | 18 +-
.../apache/samza/config/JavaSystemConfig.java | 12 +-
.../apache/samza/config/JavaTableConfig.java | 2 +-
.../samza/config/JobCoordinatorConfig.java | 4 +-
.../org/apache/samza/config/TaskConfigJava.java | 35 +-
.../container/ContainerHeartbeatClient.java | 4 +-
.../container/ContainerHeartbeatMonitor.java | 3 +
.../apache/samza/container/LocalityManager.java | 94 ++-
.../apache/samza/container/RunLoopFactory.java | 25 +-
.../SamzaContainerExceptionHandler.java | 57 --
.../apache/samza/container/TaskContextImpl.java | 31 +-
.../disk/PollingScanDiskSpaceMonitor.java | 20 +-
.../grouper/task/TaskAssignmentManager.java | 37 +-
.../container/host/StatisticsMonitorImpl.java | 20 +-
.../StreamPartitionCountMonitor.java | 10 +-
.../AbstractCoordinatorStreamManager.java | 132 ----
.../stream/CoordinatorStreamManager.java | 170 ++++++
.../stream/CoordinatorStreamSystemConsumer.java | 32 +-
.../stream/CoordinatorStreamSystemProducer.java | 37 +-
.../stream/CoordinatorStreamWriter.java | 2 +-
.../samza/execution/ExecutionPlanner.java | 30 +-
.../org/apache/samza/execution/JobGraph.java | 17 +-
.../samza/execution/JobGraphJsonGenerator.java | 4 +-
.../org/apache/samza/execution/JobNode.java | 64 +-
.../org/apache/samza/execution/StreamEdge.java | 1 +
.../apache/samza/execution/StreamManager.java | 32 +-
.../org/apache/samza/job/model/JobModel.java | 4 +-
.../samza/operators/MessageStreamImpl.java | 89 +--
.../samza/operators/OperatorSpecGraph.java | 132 ++++
.../apache/samza/operators/StreamGraphImpl.java | 315 ----------
.../apache/samza/operators/StreamGraphSpec.java | 299 +++++++++
.../org/apache/samza/operators/TableImpl.java | 3 +-
.../operators/impl/BroadcastOperatorImpl.java | 84 +++
.../operators/impl/ControlMessageSender.java | 43 +-
.../samza/operators/impl/EndOfStreamStates.java | 6 +-
.../samza/operators/impl/OperatorImpl.java | 70 ++-
.../samza/operators/impl/OperatorImplGraph.java | 110 ++--
.../operators/impl/OutputOperatorImpl.java | 5 +-
.../operators/impl/PartitionByOperatorImpl.java | 16 +-
.../operators/impl/StreamOperatorImpl.java | 3 +-
.../samza/operators/impl/WatermarkStates.java | 12 +-
.../operators/impl/WindowOperatorImpl.java | 23 +-
.../operators/spec/BroadcastOperatorSpec.java | 49 ++
.../operators/spec/FilterOperatorSpec.java | 74 +++
.../operators/spec/FlatMapOperatorSpec.java | 47 ++
.../samza/operators/spec/InputOperatorSpec.java | 21 +-
.../samza/operators/spec/JoinOperatorSpec.java | 23 +-
.../samza/operators/spec/MapOperatorSpec.java | 77 +++
.../samza/operators/spec/MergeOperatorSpec.java | 51 ++
.../samza/operators/spec/OperatorSpec.java | 29 +-
.../samza/operators/spec/OperatorSpecs.java | 85 +--
.../operators/spec/OutputOperatorSpec.java | 6 +
.../samza/operators/spec/OutputStreamImpl.java | 17 +-
.../operators/spec/PartitionByOperatorSpec.java | 29 +-
.../operators/spec/SendToTableOperatorSpec.java | 15 +-
.../samza/operators/spec/SinkOperatorSpec.java | 6 +
.../operators/spec/StreamOperatorSpec.java | 14 +-
.../spec/StreamTableJoinOperatorSpec.java | 6 +
.../operators/spec/WindowOperatorSpec.java | 22 +-
.../stream/IntermediateMessageStreamImpl.java | 4 +-
.../samza/operators/triggers/Cancellable.java | 2 +-
.../samza/operators/triggers/TriggerImpl.java | 6 +-
.../operators/util/InternalInMemoryStore.java | 6 +
.../apache/samza/operators/util/MathUtils.java | 50 --
.../apache/samza/processor/StreamProcessor.java | 238 ++++----
.../runtime/AbstractApplicationRunner.java | 41 +-
.../samza/runtime/ApplicationRunnerMain.java | 9 +
.../samza/runtime/LocalApplicationRunner.java | 77 ++-
.../samza/runtime/LocalContainerRunner.java | 30 +-
.../samza/runtime/RemoteApplicationRunner.java | 78 ++-
.../serializers/model/SamzaObjectMapper.java | 17 +-
.../standalone/PassthroughJobCoordinator.java | 37 +-
.../storage/ChangelogPartitionManager.java | 89 ---
.../samza/storage/ChangelogStreamManager.java | 154 +++++
.../apache/samza/storage/StorageRecovery.java | 63 +-
.../org/apache/samza/system/SystemAdmins.java | 71 +++
.../samza/system/inmemory/InMemoryManager.java | 179 ++++++
.../system/inmemory/InMemorySystemAdmin.java | 137 +++++
.../system/inmemory/InMemorySystemConsumer.java | 148 +++++
.../system/inmemory/InMemorySystemFactory.java | 50 ++
.../system/inmemory/InMemorySystemProducer.java | 103 ++++
.../org/apache/samza/table/TableManager.java | 53 +-
.../samza/table/caching/CachingTable.java | 199 ++++++
.../table/caching/CachingTableDescriptor.java | 173 ++++++
.../table/caching/CachingTableProvider.java | 138 +++++
.../caching/CachingTableProviderFactory.java | 34 ++
.../samza/table/caching/SupplierGauge.java | 46 ++
.../table/caching/guava/GuavaCacheTable.java | 112 ++++
.../guava/GuavaCacheTableDescriptor.java | 76 +++
.../caching/guava/GuavaCacheTableProvider.java | 92 +++
.../guava/GuavaCacheTableProviderFactory.java | 34 ++
.../samza/table/remote/CreditFunction.java | 36 ++
.../table/remote/RemoteReadWriteTable.java | 184 ++++++
.../samza/table/remote/RemoteReadableTable.java | 181 ++++++
.../table/remote/RemoteTableDescriptor.java | 194 ++++++
.../samza/table/remote/RemoteTableProvider.java | 144 +++++
.../remote/RemoteTableProviderFactory.java | 38 ++
.../samza/table/remote/TableReadFunction.java | 66 ++
.../samza/table/remote/TableWriteFunction.java | 86 +++
.../apache/samza/table/utils/SerdeUtils.java | 66 ++
.../org/apache/samza/task/AsyncRunLoop.java | 151 ++++-
.../apache/samza/task/StreamOperatorTask.java | 48 +-
.../apache/samza/task/SystemTimerScheduler.java | 154 +++++
.../org/apache/samza/task/TaskCallbackImpl.java | 22 +-
.../apache/samza/task/TaskCallbackManager.java | 11 +-
.../task/TaskCallbackTimeoutException.java | 42 --
.../org/apache/samza/task/TaskFactoryUtil.java | 42 +-
.../apache/samza/util/ClassLoaderHelper.java | 48 --
.../samza/util/EmbeddedTaggedRateLimiter.java | 136 +++++
.../java/org/apache/samza/util/MathUtil.java | 77 +++
.../samza/util/MetricsReporterLoader.java | 6 +-
.../util/SamzaUncaughtExceptionHandler.java | 69 +++
.../org/apache/samza/util/ScalaToJavaUtils.java | 41 --
.../org/apache/samza/util/ShutdownUtil.java | 74 +++
.../apache/samza/util/ThrottlingExecutor.java | 4 +-
.../apache/samza/util/ThrottlingScheduler.java | 2 +-
.../samza/zk/ScheduleAfterDebounceTime.java | 96 ++-
.../samza/zk/ZkBarrierForVersionUpgrade.java | 125 +++-
.../org/apache/samza/zk/ZkControllerImpl.java | 22 +-
.../samza/zk/ZkCoordinationUtilsFactory.java | 6 +-
.../org/apache/samza/zk/ZkJobCoordinator.java | 229 ++++---
.../samza/zk/ZkJobCoordinatorFactory.java | 2 +-
.../org/apache/samza/zk/ZkLeaderElector.java | 3 +-
.../main/java/org/apache/samza/zk/ZkUtils.java | 159 +++--
.../org/apache/samza/zk/ZkUtilsMetrics.java | 6 +
.../samza/checkpoint/CheckpointTool.scala | 23 +-
.../apache/samza/checkpoint/OffsetManager.scala | 10 +-
.../org/apache/samza/config/JobConfig.scala | 11 +
.../apache/samza/config/SerializerConfig.scala | 32 +
.../org/apache/samza/config/StorageConfig.scala | 7 +-
.../org/apache/samza/config/StreamConfig.scala | 14 +-
.../org/apache/samza/config/SystemConfig.scala | 7 +
.../org/apache/samza/config/TaskConfig.scala | 58 +-
.../org/apache/samza/container/RunLoop.scala | 4 +-
.../apache/samza/container/SamzaContainer.scala | 215 ++++---
.../samza/container/SamzaContainerMetrics.scala | 2 +
.../apache/samza/container/TaskInstance.scala | 42 +-
.../samza/coordinator/JobModelManager.scala | 169 +-----
.../stream/CoordinatorStreamSystemFactory.scala | 50 --
.../scala/org/apache/samza/job/JobRunner.scala | 52 +-
.../org/apache/samza/job/local/ProcessJob.scala | 167 +++---
.../samza/job/local/ProcessJobFactory.scala | 44 +-
.../samza/job/local/ThreadJobFactory.scala | 49 +-
.../org/apache/samza/metrics/JmxServer.scala | 1 +
.../org/apache/samza/metrics/JvmMetrics.scala | 24 +-
.../reporter/MetricsSnapshotReporter.scala | 31 +-
.../MetricsSnapshotReporterFactory.scala | 6 +-
.../apache/samza/serializers/SerdeManager.scala | 2 -
.../samza/storage/TaskStorageManager.scala | 45 +-
.../samza/system/StreamMetadataCache.scala | 25 +-
.../apache/samza/system/SystemConsumers.scala | 11 +-
.../system/chooser/BootstrappingChooser.scala | 38 +-
.../samza/system/chooser/DefaultChooser.scala | 12 +-
.../filereader/FileReaderSystemConsumer.scala | 27 +-
.../org/apache/samza/util/CommandLine.scala | 4 +-
.../samza/util/CoordinatorStreamUtil.scala | 94 +++
.../apache/samza/util/DaemonThreadFactory.scala | 39 --
.../samza/util/ExponentialSleepStrategy.scala | 17 +-
.../scala/org/apache/samza/util/FileUtil.scala | 104 ++++
.../scala/org/apache/samza/util/HttpUtil.scala | 89 +++
.../samza/util/LexicographicComparator.scala | 39 --
.../org/apache/samza/util/ScalaJavaUtil.scala | 62 ++
.../scala/org/apache/samza/util/TimerUtil.scala | 56 ++
.../org/apache/samza/util/TimerUtils.scala | 56 --
.../main/scala/org/apache/samza/util/Util.scala | 352 ++---------
.../MockClusterResourceManager.java | 39 +-
.../MockHostAwareContainerAllocator.java | 68 +++
.../TestClusterBasedJobCoordinator.java | 7 +-
.../TestContainerProcessManager.java | 138 +++--
.../TestHostAwareContainerAllocator.java | 164 ++++-
.../samza/container/TestLocalityManager.java | 26 +-
.../TestSamzaContainerExceptionHandler.java | 39 --
.../TestSamzaUncaughtExceptionHandler.java | 40 ++
.../grouper/task/TestTaskAssignmentManager.java | 25 +-
.../MockCoordinatorStreamSystemFactory.java | 6 +-
.../apache/samza/example/BroadcastExample.java | 71 ---
.../samza/example/KeyValueStoreExample.java | 131 ----
.../org/apache/samza/example/MergeExample.java | 60 --
.../samza/example/OrderShipmentJoinExample.java | 115 ----
.../samza/example/PageViewCounterExample.java | 95 ---
.../samza/example/RepartitionExample.java | 90 ---
.../org/apache/samza/example/WindowExample.java | 81 ---
.../samza/execution/TestExecutionPlanner.java | 150 ++---
.../apache/samza/execution/TestJobGraph.java | 68 +--
.../execution/TestJobGraphJsonGenerator.java | 59 +-
.../org/apache/samza/execution/TestJobNode.java | 14 +-
.../apache/samza/execution/TestStreamEdge.java | 4 +-
.../samza/execution/TestStreamManager.java | 24 +-
.../samza/operators/TestJoinOperator.java | 152 ++---
.../samza/operators/TestMessageStreamImpl.java | 55 +-
.../samza/operators/TestOperatorSpecGraph.java | 185 ++++++
.../samza/operators/TestStreamGraphImpl.java | 601 -------------------
.../samza/operators/TestStreamGraphSpec.java | 601 +++++++++++++++++++
.../data/TestOutputMessageEnvelope.java | 14 +
.../impl/TestControlMessageSender.java | 32 +-
.../samza/operators/impl/TestOperatorImpl.java | 6 +
.../operators/impl/TestOperatorImplGraph.java | 298 ++++++---
.../operators/impl/TestStreamOperatorImpl.java | 4 +-
.../operators/impl/TestWindowOperator.java | 263 ++++----
.../operators/impl/store/TestInMemoryStore.java | 16 +
.../operators/spec/OperatorSpecTestUtils.java | 141 +++++
.../samza/operators/spec/TestOperatorSpec.java | 465 ++++++++++++++
.../spec/TestPartitionByOperatorSpec.java | 165 +++++
.../operators/spec/TestWindowOperatorSpec.java | 306 +++++++++-
.../runtime/TestAbstractApplicationRunner.java | 36 +-
.../runtime/TestApplicationRunnerMain.java | 2 +
.../runtime/TestLocalApplicationRunner.java | 46 +-
.../runtime/TestRemoteApplicationRunner.java | 53 ++
.../model/TestSamzaObjectMapper.java | 33 +-
.../system/inmemory/TestInMemorySystem.java | 211 +++++++
.../apache/samza/table/TestTableManager.java | 14 +-
.../samza/table/caching/TestCachingTable.java | 299 +++++++++
.../table/remote/TestRemoteTableDescriptor.java | 244 ++++++++
.../apache/samza/task/IdentityStreamTask.java | 55 ++
.../org/apache/samza/task/TestAsyncRunLoop.java | 333 +++++-----
.../samza/task/TestSystemTimerScheduler.java | 176 ++++++
.../apache/samza/task/TestTaskFactoryUtil.java | 64 +-
.../testUtils/InvalidStreamApplication.java | 25 -
.../util/TestEmbeddedTaggedRateLimiter.java | 230 +++++++
.../org/apache/samza/util/TestMathUtils.java | 43 +-
.../org/apache/samza/util/TestShutdownUtil.java | 63 ++
.../samza/zk/TestScheduleAfterDebounceTime.java | 59 +-
.../zk/TestZkBarrierForVersionUpgrade.java | 280 ++++-----
.../apache/samza/zk/TestZkJobCoordinator.java | 39 +-
.../apache/samza/zk/TestZkLeaderElector.java | 2 +-
.../apache/samza/zk/TestZkProcessorLatch.java | 6 +-
.../java/org/apache/samza/zk/TestZkUtils.java | 119 +++-
.../samza/checkpoint/TestOffsetManager.scala | 28 +-
.../samza/config/TestSerializerConfig.scala | 57 ++
.../samza/container/TestSamzaContainer.scala | 29 +-
.../samza/container/TestTaskInstance.scala | 4 +-
.../samza/coordinator/TestJobCoordinator.scala | 77 ++-
.../TestStreamPartitionCountMonitor.scala | 14 +-
.../coordinator/server/TestHttpServer.scala | 7 +-
.../apache/samza/job/local/TestProcessJob.scala | 133 +++-
.../processor/StreamProcessorTestUtils.scala | 6 +-
.../samza/serializers/TestSerdeManager.scala | 3 +
.../samza/storage/TestTaskStorageManager.scala | 46 +-
.../samza/system/TestStreamMetadataCache.scala | 101 ++--
.../chooser/TestBootstrappingChooser.scala | 94 ++-
.../system/chooser/TestDefaultChooser.scala | 17 +-
.../samza/util/TestDaemonThreadFactory.scala | 37 --
.../org/apache/samza/util/TestFileUtil.scala | 84 +++
.../scala/org/apache/samza/util/TestUtil.scala | 100 +--
.../ElasticsearchSystemFactory.java | 2 +-
.../samza/system/hdfs/HdfsSystemAdmin.java | 2 +-
.../samza/system/hdfs/HdfsSystemConsumer.java | 2 +-
.../hdfs/partitioner/HdfsFileSystemAdapter.java | 7 +-
.../apache/samza/system/hdfs/HdfsConfig.scala | 4 +-
.../samza/system/hdfs/HdfsSystemProducer.scala | 4 +-
.../hdfs/writer/AvroDataFileHdfsWriter.scala | 16 +-
.../system/hdfs/TestHdfsSystemConsumer.java | 2 +-
.../partitioner/TestHdfsFileSystemAdapter.java | 2 +-
.../resources/partitioner/subfolder/testfile002 | 16 +
.../kafka/KafkaCheckpointLogKeySerde.java | 5 +
.../samza/system/kafka/KafkaStreamSpec.java | 18 +-
.../kafka/KafkaCheckpointManager.scala | 91 ++-
.../kafka/KafkaCheckpointManagerFactory.scala | 2 +-
.../org/apache/samza/config/KafkaConfig.scala | 24 +-
.../apache/samza/system/kafka/BrokerProxy.scala | 25 +-
.../apache/samza/system/kafka/GetOffset.scala | 4 +-
.../samza/system/kafka/KafkaSystemAdmin.scala | 72 ++-
.../system/kafka/KafkaSystemConsumer.scala | 2 +
.../samza/system/kafka/KafkaSystemFactory.scala | 10 +-
.../system/kafka/KafkaSystemProducer.scala | 4 +-
.../samza/system/kafka/TopicMetadataCache.scala | 2 +-
.../scala/org/apache/samza/util/KafkaUtil.scala | 8 +-
.../kafka/TestKafkaCheckpointManagerJava.java | 45 +-
.../samza/system/kafka/MockKafkaProducer.java | 25 +-
.../samza/system/kafka/TestKafkaStreamSpec.java | 3 +-
.../system/kafka/TestKafkaSystemAdminJava.java | 44 +-
.../kafka/TestKafkaCheckpointManager.scala | 38 +-
.../apache/samza/config/TestKafkaConfig.scala | 10 +-
.../samza/system/kafka/TestBrokerProxy.scala | 9 +-
.../system/kafka/TestKafkaSystemAdmin.scala | 54 +-
.../system/kafka/TestKafkaSystemConsumer.scala | 4 +-
.../system/kafka/TestTopicMetadataCache.scala | 32 +-
.../org/apache/samza/utils/TestKafkaUtil.scala | 7 +-
.../kv/inmemory/InMemoryKeyValueStore.scala | 14 +-
.../kv/inmemory/TestInMemoryKeyValueStore.java | 84 +++
.../samza/storage/kv/RocksDbKeyValueReader.java | 5 +-
.../samza/storage/kv/RocksDbOptionsHelper.java | 12 +-
.../RocksDbKeyValueStorageEngineFactory.scala | 3 +
.../samza/storage/kv/RocksDbKeyValueStore.scala | 76 ++-
.../kv/TestRocksDbKeyValueStoreJava.java | 140 +++++
.../storage/kv/TestRocksDbKeyValueStore.scala | 6 +-
.../kv/BaseLocalStoreBackedTableProvider.java | 54 +-
.../kv/LocalStoreBackedReadWriteTable.java | 10 +-
.../kv/LocalStoreBackedReadableTable.java | 8 +-
.../samza/storage/kv/AccessLoggedStore.scala | 9 +-
.../kv/BaseKeyValueStorageEngineFactory.scala | 8 +-
.../apache/samza/storage/kv/CachedStore.scala | 4 +
.../storage/kv/KeyValueStorageEngine.scala | 52 +-
.../kv/KeyValueStorageEngineMetrics.scala | 21 +-
.../samza/storage/kv/KeyValueStoreMetrics.scala | 5 +-
.../apache/samza/storage/kv/LoggedStore.scala | 3 +
.../storage/kv/NullSafeKeyValueStore.scala | 14 +-
.../storage/kv/SerializedKeyValueStore.scala | 15 +
.../TestLocalBaseStoreBackedTableProvider.java | 6 +-
.../samza/storage/kv/MockKeyValueStore.scala | 4 +
.../storage/kv/TestKeyValueStorageEngine.scala | 6 +-
.../samza/logging/log4j/StreamAppender.java | 69 ++-
.../samza/logging/log4j/MockSystemAdmin.java | 74 +++
.../samza/logging/log4j/MockSystemFactory.java | 2 +-
.../samza/logging/log4j/TestStreamAppender.java | 57 ++
.../samza/monitor/LocalStoreMonitorConfig.java | 4 +-
.../org/apache/samza/monitor/MonitorConfig.java | 8 +
.../org/apache/samza/monitor/MonitorLoader.java | 4 +-
.../samza/monitor/SamzaMonitorService.java | 8 +-
.../apache/samza/rest/SamzaRestApplication.java | 4 +-
.../samza/rest/proxy/job/AbstractJobProxy.java | 8 +-
.../rest/proxy/job/SimpleYarnJobProxy.java | 5 +-
.../samza/rest/proxy/task/SamzaTaskProxy.java | 14 +-
.../rest/proxy/task/SamzaTaskProxyFactory.java | 7 +-
.../samza/rest/resources/TasksResource.java | 9 +-
.../samza/monitor/TestLocalStoreMonitor.java | 84 +--
.../apache/samza/sql/avro/AvroRelConverter.java | 239 +++++---
.../samza/sql/avro/AvroTypeFactoryImpl.java | 9 +-
.../samza/sql/data/SamzaSqlCompositeKey.java | 82 +++
.../sql/data/SamzaSqlExecutionContext.java | 24 +-
.../samza/sql/data/SamzaSqlRelMessage.java | 175 ++++--
.../org/apache/samza/sql/fn/FlattenUdf.java | 2 +-
.../org/apache/samza/sql/fn/RegexMatchUdf.java | 39 ++
.../sql/impl/ConfigBasedIOResolverFactory.java | 125 ++++
.../impl/ConfigBasedSourceResolverFactory.java | 71 ---
.../samza/sql/impl/ConfigBasedUdfResolver.java | 13 +-
.../interfaces/RelSchemaProviderFactory.java | 3 +-
.../interfaces/SamzaRelConverterFactory.java | 7 +-
.../samza/sql/interfaces/SourceResolver.java | 34 --
.../sql/interfaces/SourceResolverFactory.java | 36 --
.../samza/sql/interfaces/SqlIOConfig.java | 136 +++++
.../samza/sql/interfaces/SqlIOResolver.java | 45 ++
.../sql/interfaces/SqlIOResolverFactory.java | 36 ++
.../sql/interfaces/SqlSystemStreamConfig.java | 74 ---
.../apache/samza/sql/planner/QueryPlanner.java | 45 +-
.../sql/planner/SamzaSqlScalarFunctionImpl.java | 7 +-
.../sql/runner/SamzaSqlApplicationConfig.java | 83 +--
.../sql/runner/SamzaSqlApplicationRunner.java | 23 +-
.../SamzaSqlRelMessageSerdeFactory.java | 67 +++
.../SamzaSqlRelRecordSerdeFactory.java | 67 +++
.../samza/sql/testutil/SamzaSqlQueryParser.java | 67 +--
.../samza/sql/translator/FilterTranslator.java | 47 +-
.../samza/sql/translator/JoinTranslator.java | 294 +++++++++
.../translator/LogicalAggregateTranslator.java | 102 ++++
.../samza/sql/translator/ProjectTranslator.java | 71 ++-
.../samza/sql/translator/QueryTranslator.java | 103 +++-
.../SamzaSqlRelMessageJoinFunction.java | 121 ++++
.../samza/sql/translator/ScanTranslator.java | 52 +-
.../samza/sql/translator/TranslatorContext.java | 79 ++-
.../apache/samza/sql/TestQueryTranslator.java | 103 ----
.../sql/TestSamzaSqlApplicationConfig.java | 92 ---
.../samza/sql/TestSamzaSqlFileParser.java | 58 --
.../samza/sql/TestSamzaSqlQueryParser.java | 70 ---
.../samza/sql/TestSamzaSqlRelMessage.java | 46 --
.../samza/sql/TestSamzaSqlRelMessageSerde.java | 102 ++++
.../samza/sql/TestSamzaSqlRelRecordSerde.java | 86 +++
.../samza/sql/avro/TestAvroRelConversion.java | 132 +++-
.../samza/sql/avro/schemas/AddressRecord.java | 52 ++
.../apache/samza/sql/avro/schemas/Company.avsc | 39 ++
.../apache/samza/sql/avro/schemas/Company.java | 52 ++
.../sql/avro/schemas/EnrichedPageView.avsc | 81 +++
.../sql/avro/schemas/EnrichedPageView.java | 60 ++
.../org/apache/samza/sql/avro/schemas/Kind.java | 30 +
.../apache/samza/sql/avro/schemas/PageView.avsc | 39 ++
.../apache/samza/sql/avro/schemas/PageView.java | 52 ++
.../samza/sql/avro/schemas/PageViewCount.avsc | 45 ++
.../samza/sql/avro/schemas/PageViewCount.java | 56 ++
.../samza/sql/avro/schemas/PhoneNumber.java | 50 ++
.../apache/samza/sql/avro/schemas/Profile.avsc | 149 +++++
.../apache/samza/sql/avro/schemas/Profile.java | 72 +++
.../samza/sql/avro/schemas/SimpleRecord.avsc | 2 +-
.../samza/sql/avro/schemas/SimpleRecord.java | 2 +-
.../samza/sql/avro/schemas/StreetNumRecord.java | 48 ++
.../samza/sql/data/TestSamzaSqlRelMessage.java | 46 ++
.../samza/sql/e2e/TestSamzaSqlEndToEnd.java | 137 -----
.../apache/samza/sql/e2e/TestSamzaSqlTable.java | 69 +++
.../runner/TestSamzaSqlApplicationConfig.java | 95 +++
.../runner/TestSamzaSqlApplicationRunner.java | 56 ++
.../samza/sql/system/SimpleSystemAdmin.java | 11 +-
.../samza/sql/system/TestAvroSystemFactory.java | 186 +++++-
.../samza/sql/testutil/MyTestArrayUdf.java | 5 +-
.../apache/samza/sql/testutil/MyTestUdf.java | 2 +-
.../samza/sql/testutil/SamzaSqlTestConfig.java | 97 ++-
.../sql/testutil/TestIOResolverFactory.java | 196 ++++++
.../sql/testutil/TestSamzaSqlFileParser.java | 58 ++
.../sql/testutil/TestSamzaSqlQueryParser.java | 75 +++
.../sql/translator/TestFilterTranslator.java | 136 +++++
.../sql/translator/TestJoinTranslator.java | 191 ++++++
.../sql/translator/TestProjectTranslator.java | 289 +++++++++
.../sql/translator/TestQueryTranslator.java | 596 ++++++++++++++++++
.../TestSamzaSqlRelMessageJoinFunction.java | 118 ++++
.../sql/translator/TranslatorTestBase.java | 72 +++
samza-sql/src/test/resources/log4j.xml | 6 +
samza-test/src/main/config/join/README | 8 +-
.../example/AppWithGlobalConfigExample.java | 86 +++
.../apache/samza/example/BroadcastExample.java | 70 +++
.../samza/example/KeyValueStoreExample.java | 138 +++++
.../org/apache/samza/example/MergeExample.java | 62 ++
.../samza/example/OrderShipmentJoinExample.java | 121 ++++
.../samza/example/PageViewCounterExample.java | 100 +++
.../samza/example/RepartitionExample.java | 96 +++
.../org/apache/samza/example/WindowExample.java | 86 +++
.../samza/test/framework/StreamAssert.java | 181 ++++++
samza-test/src/main/python/configs/tests.json | 2 +-
.../performance/TestKeyValuePerformance.scala | 6 +-
.../samza/processor/TestZkStreamProcessor.java | 11 +-
.../processor/TestZkStreamProcessorBase.java | 8 +-
.../TestZkStreamProcessorFailures.java | 8 +-
.../EndOfStreamIntegrationTest.java | 8 +-
.../WatermarkIntegrationTest.java | 7 +-
.../samza/test/operator/BroadcastAssertApp.java | 59 ++
.../test/operator/RepartitionJoinWindowApp.java | 80 ++-
.../test/operator/RepartitionWindowApp.java | 72 +++
.../samza/test/operator/SessionWindowApp.java | 21 +-
...StreamApplicationIntegrationTestHarness.java | 21 +-
.../operator/TestRepartitionJoinWindowApp.java | 120 +++-
.../test/operator/TestRepartitionWindowApp.java | 90 +++
.../samza/test/operator/TumblingWindowApp.java | 20 +-
.../samza/test/operator/data/PageView.java | 63 +-
.../test/processor/SharedContextFactories.java | 117 ++++
.../test/processor/TestStreamApplication.java | 148 +++++
.../test/processor/TestStreamProcessor.java | 9 +-
.../processor/TestZkLocalApplicationRunner.java | 363 +++++------
.../test/samzasql/TestSamzaSqlEndToEnd.java | 469 +++++++++++++++
.../apache/samza/test/table/TestLocalTable.java | 257 +++++---
.../samza/test/table/TestRemoteTable.java | 248 ++++++++
.../apache/samza/test/timer/TestTimerApp.java | 87 +++
.../org/apache/samza/test/timer/TimerTest.java | 51 ++
.../samza/storage/kv/TestKeyValueStores.scala | 240 ++++----
.../AbstractIntegrationTestHarness.scala | 8 +-
.../AbstractKafkaServerTestHarness.scala | 11 +-
.../harness/AbstractZookeeperTestHarness.scala | 10 +-
.../test/integration/StreamTaskTestUtil.scala | 39 +-
.../integration/TestShutdownStatefulTask.scala | 2 +-
.../test/integration/TestStatefulTask.scala | 4 +-
samza-tools/config/bench-log4j.xml | 35 ++
samza-tools/config/eh-bench.properties | 26 +
samza-tools/scripts/eh-consumer.sh | 2 +-
samza-tools/scripts/generate-kafka-events.sh | 2 +-
samza-tools/scripts/samza-sql-console.sh | 2 +-
samza-tools/scripts/system-consumer-bench.sh | 34 ++
.../scripts/system-consumer-with-samza-bench.sh | 34 ++
samza-tools/scripts/system-producer-bench.sh | 34 ++
.../tools/ConsoleLoggingSystemFactory.java | 27 +-
.../samza/tools/EventHubConsoleConsumer.java | 64 +-
.../apache/samza/tools/GenerateKafkaEvents.java | 4 +-
.../org/apache/samza/tools/SamzaSqlConsole.java | 40 +-
.../tools/avro/AvroSchemaGenRelConverter.java | 4 +-
.../tools/benchmark/AbstractSamzaBench.java | 153 +++++
.../benchmark/ConfigBasedSspGrouperFactory.java | 87 +++
.../tools/benchmark/SystemConsumerBench.java | 91 +++
.../benchmark/SystemConsumerWithSamzaBench.java | 117 ++++
.../tools/benchmark/SystemProducerBench.java | 124 ++++
.../tools/json/JsonRelConverterFactory.java | 4 +-
.../apache/samza/tools/udf/RegexMatchUdf.java | 40 --
.../job/yarn/YarnClusterResourceManager.java | 38 +-
.../samza/validation/YarnJobValidationTool.java | 15 +-
.../webapp/ApplicationMasterRestClient.java | 111 ++++
.../apache/samza/job/yarn/ClientHelper.scala | 54 +-
.../yarn/SamzaAppMasterSecurityManager.scala | 31 +-
.../yarn/SamzaContainerSecurityManager.scala | 34 +-
.../apache/samza/job/yarn/YarnContainer.scala | 4 +-
.../org/apache/samza/job/yarn/YarnJob.scala | 14 +-
.../webapp/ApplicationMasterRestServlet.scala | 76 ++-
.../webapp/TestApplicationMasterRestClient.java | 330 ++++++++++
.../TestYarnContainerHeartbeatServlet.java | 8 +-
.../samza/job/yarn/TestClientHelper.scala | 36 +-
.../yarn/TestSamzaYarnAppMasterService.scala | 16 +-
settings.gradle | 4 +-
sonar-project.properties | 6 +-
565 files changed, 25625 insertions(+), 8630 deletions(-)
----------------------------------------------------------------------