You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by bo...@apache.org on 2018/09/18 23:01:07 UTC
[10/47] samza git commit: Merge branch 'master' of
https://github.com/apache/samza
Merge branch 'master' of https://github.com/apache/samza
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/67e611ee
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/67e611ee
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/67e611ee
Branch: refs/heads/NewKafkaSystemConsumer
Commit: 67e611ee47f0747b81c0de5d061f9888145b605a
Parents: dd39d08 b00ebd2
Author: Boris S <bo...@apache.org>
Authored: Wed Jan 10 09:50:40 2018 -0800
Committer: Boris S <bo...@apache.org>
Committed: Wed Jan 10 09:50:40 2018 -0800
----------------------------------------------------------------------
RELEASE.md | 4 +-
bin/merge-pull-request.py | 28 +-
build.gradle | 96 +++++-
docs/README.md | 4 +-
docs/_config.yml | 4 +-
docs/_docs/replace-versioned.sh | 3 +
docs/_layouts/default.html | 1 +
docs/archive/index.html | 8 +
.../documentation/hadoop/multi_stage_batch.png | Bin 0 -> 65364 bytes
.../hadoop/unified_batch_streaming.png | Bin 0 -> 17371 bytes
.../azure/eventhub_send_methods.png | Bin 0 -> 52427 bytes
.../documentation/versioned/aws/kinesis.md | 104 +++++++
.../documentation/versioned/azure/eventhubs.md | 197 ++++++++++++
.../versioned/container/metrics-table.html | 12 +
.../documentation/versioned/hadoop/consumer.md | 110 +++++++
.../documentation/versioned/hadoop/overview.md | 46 +++
.../documentation/versioned/hadoop/producer.md | 68 ++++
.../documentation/versioned/hdfs/consumer.md | 110 -------
.../documentation/versioned/hdfs/producer.md | 70 -----
docs/learn/documentation/versioned/index.html | 27 +-
.../versioned/jobs/configuration-table.html | 128 ++++++++
.../versioned/yarn/yarn-host-affinity.md | 2 +-
.../versioned/yarn/yarn-security.md | 1 -
.../versioned/deploy-samza-job-from-hdfs.md | 2 +-
.../tutorials/versioned/deploy-samza-to-CDH.md | 4 +-
.../versioned/hello-samza-high-level-code.md | 87 +++---
.../versioned/hello-samza-high-level-yarn.md | 2 +-
.../versioned/hello-samza-high-level-zk.md | 2 +-
docs/learn/tutorials/versioned/index.md | 3 +
.../versioned/remote-debugging-samza.md | 2 +-
.../versioned/run-in-multi-node-yarn.md | 4 +-
.../versioned/samza-rest-getting-started.md | 2 +-
docs/learn/tutorials/versioned/samza-sql.md | 123 ++++++++
docs/learn/tutorials/versioned/samza-tools.md | 109 +++++++
docs/startup/download/index.md | 24 +-
docs/startup/hello-samza/versioned/index.md | 2 +-
docs/startup/preview/index.md | 135 ++++++--
.../startup/releases/versioned/release-notes.md | 82 +++++
gradle.properties | 2 +-
gradle/dependency-versions.gradle | 1 +
.../java/org/apache/samza/operators/KV.java | 2 +-
.../apache/samza/operators/MessageStream.java | 69 ++++-
.../org/apache/samza/operators/StreamGraph.java | 48 ++-
.../apache/samza/operators/TableDescriptor.java | 73 +++++
.../functions/StreamTableJoinFunction.java | 59 ++++
.../org/apache/samza/serializers/KVSerde.java | 8 +-
.../table/LocalStoreBackedTableProvider.java | 37 +++
.../org/apache/samza/table/ReadWriteTable.java | 75 +++++
.../org/apache/samza/table/ReadableTable.java | 61 ++++
.../main/java/org/apache/samza/table/Table.java | 31 ++
.../org/apache/samza/table/TableProvider.java | 57 ++++
.../samza/table/TableProviderFactory.java | 35 +++
.../java/org/apache/samza/table/TableSpec.java | 125 ++++++++
.../java/org/apache/samza/task/TaskContext.java | 6 +-
.../kinesis/KinesisAWSCredentialsProvider.java | 69 +++++
.../samza/system/kinesis/KinesisConfig.java | 287 +++++++++++++++++
.../system/kinesis/KinesisSystemAdmin.java | 124 ++++++++
.../system/kinesis/KinesisSystemFactory.java | 87 ++++++
.../KinesisIncomingMessageEnvelope.java | 62 ++++
.../consumer/KinesisRecordProcessor.java | 208 +++++++++++++
.../KinesisRecordProcessorListener.java | 51 +++
.../kinesis/consumer/KinesisSystemConsumer.java | 256 +++++++++++++++
.../consumer/KinesisSystemConsumerOffset.java | 107 +++++++
.../consumer/NoAvailablePartitionException.java | 38 +++
.../system/kinesis/consumer/SSPAllocator.java | 73 +++++
.../metrics/KinesisSystemConsumerMetrics.java | 106 +++++++
.../system/kinesis/metrics/SamzaHistogram.java | 63 ++++
.../TestKinesisAWSCredentialsProvider.java | 60 ++++
.../samza/system/kinesis/TestKinesisConfig.java | 132 ++++++++
.../kinesis/TestKinesisSystemFactory.java | 115 +++++++
.../consumer/TestKinesisRecordProcessor.java | 301 ++++++++++++++++++
.../consumer/TestKinesisSystemConsumer.java | 270 ++++++++++++++++
.../TestKinesisSystemConsumerOffset.java | 48 +++
.../kinesis/consumer/TestSSPAllocator.java | 127 ++++++++
.../azure/AzureCheckpointManager.java | 31 +-
.../azure/AzureCheckpointManagerFactory.java | 4 +-
.../samza/system/eventhub/EventHubConfig.java | 71 ++++-
.../eventhub/SamzaEventHubClientManager.java | 4 +-
.../eventhub/admin/EventHubSystemAdmin.java | 17 +-
.../consumer/EventHubSystemConsumer.java | 77 +++--
.../producer/EventHubSystemProducer.java | 103 +++---
.../azure/ITestAzureCheckpointManager.java | 2 +-
.../MockEventHubClientManagerFactory.java | 18 ++
.../eventhub/admin/TestEventHubSystemAdmin.java | 15 +-
.../consumer/ITestEventHubSystemConsumer.java | 5 +-
.../consumer/TestEventHubSystemConsumer.java | 19 +-
.../producer/ITestEventHubSystemProducer.java | 2 +
.../producer/TestEventHubSystemProducer.java | 34 +-
.../AbstractContainerAllocator.java | 17 +-
.../clustermanager/ClusterResourceManager.java | 26 +-
.../clustermanager/ContainerProcessManager.java | 74 ++++-
.../clustermanager/SamzaApplicationState.java | 8 +-
.../samza/clustermanager/SamzaResource.java | 8 +
.../apache/samza/config/JavaStorageConfig.java | 5 +-
.../apache/samza/config/JavaTableConfig.java | 87 ++++++
.../container/ContainerHeartbeatClient.java | 20 +-
.../container/ContainerHeartbeatMonitor.java | 11 +-
.../apache/samza/container/TaskContextImpl.java | 24 +-
.../AllSspToSingleTaskGrouperFactory.java | 50 ++-
.../samza/execution/ExecutionPlanner.java | 5 +
.../org/apache/samza/execution/JobGraph.java | 16 +
.../samza/execution/JobGraphJsonGenerator.java | 60 +++-
.../org/apache/samza/execution/JobNode.java | 54 +++-
.../samza/operators/BaseTableDescriptor.java | 94 ++++++
.../samza/operators/MessageStreamImpl.java | 35 ++-
.../apache/samza/operators/StreamGraphImpl.java | 46 ++-
.../org/apache/samza/operators/TableImpl.java | 40 +++
.../samza/operators/impl/InputOperatorImpl.java | 2 +-
.../samza/operators/impl/OperatorImpl.java | 16 +-
.../samza/operators/impl/OperatorImplGraph.java | 44 ++-
.../operators/impl/OutputOperatorImpl.java | 2 +-
.../operators/impl/PartitionByOperatorImpl.java | 7 +-
.../operators/impl/SendToTableOperatorImpl.java | 71 +++++
.../impl/StreamTableJoinOperatorImpl.java | 82 +++++
.../samza/operators/spec/InputOperatorSpec.java | 10 +-
.../samza/operators/spec/OperatorSpec.java | 12 +-
.../samza/operators/spec/OperatorSpecs.java | 44 ++-
.../samza/operators/spec/OutputStreamImpl.java | 10 +-
.../operators/spec/SendToTableOperatorSpec.java | 65 ++++
.../spec/StreamTableJoinOperatorSpec.java | 67 ++++
.../stream/IntermediateMessageStreamImpl.java | 13 +
.../samza/runtime/LocalContainerRunner.java | 9 +-
.../samza/runtime/RemoteApplicationRunner.java | 7 +-
.../standalone/PassthroughJobCoordinator.java | 6 +-
.../org/apache/samza/table/TableManager.java | 153 +++++++++
.../org/apache/samza/zk/ZkJobCoordinator.java | 30 +-
.../main/java/org/apache/samza/zk/ZkUtils.java | 2 +-
.../org/apache/samza/config/JobConfig.scala | 1 +
.../apache/samza/container/SamzaContainer.scala | 75 ++---
.../apache/samza/container/TaskInstance.scala | 40 ++-
.../samza/coordinator/JobModelManager.scala | 23 +-
.../org/apache/samza/metrics/JvmMetrics.scala | 28 +-
.../MockClusterResourceManager.java | 19 +-
.../MockClusterResourceManagerCallback.java | 10 +
.../clustermanager/TestContainerAllocator.java | 55 ----
.../TestContainerProcessManager.java | 101 +++---
.../TestHostAwareContainerAllocator.java | 56 ----
.../samza/config/TestJavaStorageConfig.java | 13 +
.../samza/config/TestJavaTableConfig.java | 58 ++++
.../stream/TestAllSspToSingleTaskGrouper.java | 125 ++++++++
.../execution/TestJobGraphJsonGenerator.java | 75 +++++
.../samza/operators/TestMessageStreamImpl.java | 70 ++++-
.../samza/operators/TestStreamGraphImpl.java | 25 +-
.../samza/operators/impl/TestOperatorImpl.java | 1 +
.../impl/TestStreamTableJoinOperatorImpl.java | 101 ++++++
.../apache/samza/table/TestTableManager.java | 176 +++++++++++
.../org/apache/samza/task/TestAsyncRunLoop.java | 14 +-
.../kafka/KafkaCheckpointManager.scala | 2 +-
.../system/kafka/KafkaSystemProducer.scala | 135 ++++----
.../kafka/TestKafkaCheckpointManager.scala | 1 -
.../system/kafka/TestKafkaSystemProducer.scala | 39 ++-
.../kv/inmemory/InMemoryTableDescriptor.java | 59 ++++
.../kv/inmemory/InMemoryTableProvider.java | 65 ++++
.../inmemory/InMemoryTableProviderFactory.java | 33 ++
.../inmemory/TestInMemoryTableDescriptor.java | 48 +++
.../kv/inmemory/TestInMemoryTableProvider.java | 65 ++++
.../storage/kv/RocksDbTableDescriptor.java | 232 ++++++++++++++
.../samza/storage/kv/RocksDbTableProvider.java | 64 ++++
.../storage/kv/RocksDbTableProviderFactory.java | 31 ++
.../samza/storage/kv/RocksDbKeyValueStore.scala | 12 +-
.../storage/kv/TestRocksDbTableDescriptor.java | 87 ++++++
.../storage/kv/TestRocksDbTableProvider.java | 66 ++++
.../kv/BaseLocalStoreBackedTableDescriptor.java | 56 ++++
.../kv/BaseLocalStoreBackedTableProvider.java | 92 ++++++
.../kv/LocalStoreBackedReadWriteTable.java | 68 ++++
.../kv/LocalStoreBackedReadableTable.java | 61 ++++
.../samza/storage/kv/AccessLoggedStore.scala | 4 +-
.../storage/kv/KeyValueStorageEngine.scala | 14 +-
.../kv/KeyValueStorageEngineMetrics.scala | 4 +-
.../apache/samza/storage/kv/LoggedStore.scala | 4 +-
.../storage/kv/NullSafeKeyValueStore.scala | 4 +-
.../storage/kv/SerializedKeyValueStore.scala | 4 +-
.../TestLocalBaseStoreBackedTableProvider.java | 77 +++++
.../storage/kv/TestKeyValueStorageEngine.scala | 16 +
.../samza/logging/log4j/StreamAppender.java | 110 ++++++-
.../logging/log4j/StreamAppenderMetrics.java | 43 +++
.../samza/logging/log4j/MockSystemProducer.java | 12 +-
.../samza/logging/log4j/TestStreamAppender.java | 144 +++++++--
.../src/main/visualizer/js/planToDagre.js | 6 +-
.../apache/samza/sql/avro/AvroRelConverter.java | 6 +-
samza-test/src/main/config/join/README | 8 +-
samza-test/src/main/python/configs/tests.json | 2 +-
.../test/processor/TestStreamProcessor.java | 9 +-
.../processor/TestZkLocalApplicationRunner.java | 152 +++++++--
.../apache/samza/test/table/TestLocalTable.java | 304 ++++++++++++++++++
.../apache/samza/test/table/TestTableData.java | 200 ++++++++++++
.../samza/test/util/ArraySystemConsumer.java | 4 +-
.../samza/test/util/SimpleSystemAdmin.java | 26 +-
samza-tools/config/eh-consumer-log4j.xml | 35 +++
.../config/generate-kafka-events-log4j.xml | 35 +++
samza-tools/config/samza-sql-console-log4j.xml | 35 +++
samza-tools/scripts/eh-consumer.sh | 34 ++
samza-tools/scripts/generate-kafka-events.sh | 34 ++
samza-tools/scripts/samza-sql-console.sh | 34 ++
.../apache/samza/tools/CommandLineHelper.java | 42 +++
.../tools/ConsoleLoggingSystemFactory.java | 126 ++++++++
.../samza/tools/EventHubConsoleConsumer.java | 120 +++++++
.../apache/samza/tools/GenerateKafkaEvents.java | 205 ++++++++++++
.../samza/tools/RandomValueGenerator.java | 87 ++++++
.../org/apache/samza/tools/SamzaSqlConsole.java | 188 +++++++++++
.../tools/avro/AvroSchemaGenRelConverter.java | 94 ++++++
.../avro/AvroSchemaGenRelConverterFactory.java | 43 +++
.../samza/tools/avro/AvroSerDeFactory.java | 96 ++++++
.../tools/json/JsonRelConverterFactory.java | 93 ++++++
.../samza/tools/schemas/PageViewEvent.avsc | 51 +++
.../samza/tools/schemas/PageViewEvent.java | 60 ++++
.../samza/tools/schemas/ProfileChangeEvent.avsc | 51 +++
.../samza/tools/schemas/ProfileChangeEvent.java | 60 ++++
.../apache/samza/tools/udf/RegexMatchUdf.java | 40 +++
samza-tools/src/main/resources/log4j.xml | 43 +++
.../org/apache/samza/job/yarn/YarnAppState.java | 4 +-
.../job/yarn/YarnClusterResourceManager.java | 310 +++++++++++++++++--
.../samza/job/yarn/YarnContainerRunner.java | 272 ----------------
settings.gradle | 6 +-
214 files changed, 10944 insertions(+), 1302 deletions(-)
----------------------------------------------------------------------