You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by bo...@apache.org on 2018/09/18 23:01:06 UTC
[09/47] samza git commit: Merge branch 'master' of
https://github.com/apache/samza
Merge branch 'master' of https://github.com/apache/samza
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/dd39d089
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/dd39d089
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/dd39d089
Branch: refs/heads/NewKafkaSystemConsumer
Commit: dd39d089437905abb93aa1074474b28a18292a54
Parents: 1ad58d4 7a2e192
Author: Boris S <bo...@apache.org>
Authored: Wed Nov 22 11:46:37 2017 -0800
Committer: Boris S <bo...@apache.org>
Committed: Wed Nov 22 11:46:37 2017 -0800
----------------------------------------------------------------------
.gitignore | 3 +-
NOTICE | 2 -
build.gradle | 35 +-
.../versioned/jobs/configuration-table.html | 14 +-
gradle/dependency-versions.gradle | 4 +-
.../samza/checkpoint/CheckpointManager.java | 6 +-
.../samza/container/SamzaContainerContext.java | 7 +-
.../operators/functions/WatermarkFunction.java | 7 +-
.../org/apache/samza/sql/udfs/ScalarUdf.java | 48 ++
.../org/apache/samza/system/ControlMessage.java | 31 +
.../apache/samza/system/WatermarkMessage.java | 26 +
.../azure/AzureCheckpointManager.java | 236 +++++++
.../azure/AzureCheckpointManagerFactory.java | 33 +
.../checkpoint/azure/TaskCheckpointEntity.java | 43 ++
.../org/apache/samza/config/AzureConfig.java | 2 +-
.../coordinator/AzureCoordinationUtils.java | 2 +-
.../samza/coordinator/AzureJobCoordinator.java | 2 +-
.../producer/EventHubSystemProducer.java | 2 +-
.../azure/ITestAzureCheckpointManager.java | 181 +++++
.../consumer/TestEventHubSystemConsumer.java | 56 +-
.../producer/SwapFirstLastByteInterceptor.java | 36 +
.../producer/TestEventHubSystemProducer.java | 57 ++
.../apache/samza/PartitionChangeException.java | 31 +
.../ClusterBasedJobCoordinator.java | 108 ++-
.../clustermanager/ContainerProcessManager.java | 26 +-
.../clustermanager/SamzaApplicationState.java | 7 +-
.../org/apache/samza/config/TaskConfigJava.java | 2 +-
.../StreamPartitionCountMonitor.java | 116 ++--
.../org/apache/samza/execution/StreamEdge.java | 1 +
.../apache/samza/operators/StreamGraphImpl.java | 7 +
.../samza/operators/impl/OperatorImpl.java | 68 +-
.../operators/impl/PartitionByOperatorImpl.java | 7 +-
.../operators/impl/WindowOperatorImpl.java | 13 +
.../samza/runtime/LocalApplicationRunner.java | 2 +-
.../apache/samza/storage/StorageRecovery.java | 2 +-
.../samza/checkpoint/CheckpointTool.scala | 15 +-
.../org/apache/samza/config/JobConfig.scala | 2 -
.../org/apache/samza/config/StorageConfig.scala | 8 +
.../org/apache/samza/config/StreamConfig.scala | 30 +-
.../org/apache/samza/config/TaskConfig.scala | 5 +
.../apache/samza/container/SamzaContainer.scala | 2 +-
.../samza/coordinator/JobModelManager.scala | 49 +-
.../samza/storage/TaskStorageManager.scala | 54 +-
.../main/scala/org/apache/samza/util/Util.scala | 1 -
.../MockClusterResourceManagerFactory.java | 32 +
.../clustermanager/MockContainerListener.java | 1 +
.../TestClusterBasedJobCoordinator.java | 108 +++
.../clustermanager/TestContainerAllocator.java | 12 +-
.../TestContainerProcessManager.java | 22 +-
.../TestHostAwareContainerAllocator.java | 16 +-
.../apache/samza/config/TestTaskConfigJava.java | 20 +
.../coordinator/JobModelManagerTestUtil.java | 4 +-
.../samza/coordinator/TestJobModelManager.java | 24 +-
.../apache/samza/execution/TestStreamEdge.java | 1 +
.../samza/operators/TestStreamGraphImpl.java | 38 ++
.../samza/operators/TestWindowOperator.java | 565 ----------------
.../operators/impl/TestOperatorImplGraph.java | 10 +-
.../operators/impl/TestWindowOperator.java | 677 +++++++++++++++++++
.../samza/storage/MockSystemConsumer.java | 59 --
.../apache/samza/storage/MockSystemFactory.java | 45 --
.../samza/storage/TestStorageRecovery.java | 37 +-
.../apache/samza/system/MockSystemFactory.java | 181 +++++
.../zk/TestZkBarrierForVersionUpgrade.java | 23 +-
.../samza/checkpoint/TestCheckpointTool.scala | 23 +-
.../samza/container/TestSamzaContainer.scala | 13 +-
.../samza/container/TestTaskInstance.scala | 10 +-
.../samza/coordinator/TestJobCoordinator.scala | 65 +-
.../TestStreamPartitionCountMonitor.scala | 82 ++-
.../processor/StreamProcessorTestUtils.scala | 3 +-
.../samza/serializers/TestCheckpointSerde.scala | 8 +
.../samza/storage/TestTaskStorageManager.scala | 210 ++++--
.../TestRangeSystemStreamPartitionMatcher.scala | 1 -
.../TestRegexSystemStreamPartitionMatcher.scala | 1 -
.../scala/org/apache/samza/util/TestUtil.scala | 16 +
.../checkpoint/kafka/KafkaCheckpointLogKey.java | 110 +++
.../kafka/KafkaCheckpointLogKeySerde.java | 68 ++
.../samza/system/kafka/KafkaStreamSpec.java | 4 +
.../kafka/KafkaCheckpointLogKey.scala | 171 -----
.../kafka/KafkaCheckpointManager.scala | 385 ++++++-----
.../kafka/KafkaCheckpointManagerFactory.scala | 81 +--
.../system/kafka/KafkaSystemConsumer.scala | 8 +-
.../kafka/KafkaSystemConsumerMetrics.scala | 2 -
.../kafka/TestKafkaCheckpointLogKeySerde.java | 53 ++
.../kafka/TestKafkaCheckpointManagerJava.java | 247 +++++++
.../kafka/TeskKafkaCheckpointLogKey.scala | 61 --
.../kafka/TestKafkaCheckpointManager.scala | 388 ++++-------
.../system/kafka/TestKafkaSystemAdmin.scala | 2 +-
.../samza/storage/kv/RocksDbKeyValueReader.java | 3 +-
.../apache/samza/monitor/LocalStoreMonitor.java | 1 +
.../samza/monitor/LocalStoreMonitorMetrics.java | 4 +
.../apache/samza/sql/avro/AvroRelConverter.java | 183 +++++
.../samza/sql/avro/AvroRelConverterFactory.java | 44 ++
.../samza/sql/avro/AvroRelSchemaProvider.java | 28 +
.../samza/sql/avro/AvroTypeFactoryImpl.java | 132 ++++
...ConfigBasedAvroRelSchemaProviderFactory.java | 63 ++
.../org/apache/samza/sql/data/Expression.java | 38 ++
.../samza/sql/data/RexToJavaCompiler.java | 224 ++++++
.../sql/data/SamzaSqlExecutionContext.java | 61 ++
.../samza/sql/data/SamzaSqlRelMessage.java | 123 ++++
.../org/apache/samza/sql/fn/FlattenUdf.java | 36 +
.../impl/ConfigBasedSourceResolverFactory.java | 71 ++
.../samza/sql/impl/ConfigBasedUdfResolver.java | 97 +++
.../samza/sql/interfaces/RelSchemaProvider.java | 36 +
.../interfaces/RelSchemaProviderFactory.java | 33 +
.../samza/sql/interfaces/SamzaRelConverter.java | 46 ++
.../interfaces/SamzaRelConverterFactory.java | 39 ++
.../samza/sql/interfaces/SourceResolver.java | 34 +
.../sql/interfaces/SourceResolverFactory.java | 36 +
.../sql/interfaces/SqlSystemStreamConfig.java | 74 ++
.../samza/sql/interfaces/UdfMetadata.java | 61 ++
.../samza/sql/interfaces/UdfResolver.java | 35 +
.../org/apache/samza/sql/planner/Checker.java | 93 +++
.../apache/samza/sql/planner/QueryPlanner.java | 153 +++++
.../sql/planner/SamzaSqlOperatorTable.java | 101 +++
.../sql/planner/SamzaSqlScalarFunctionImpl.java | 84 +++
.../sql/planner/SamzaSqlUdfOperatorTable.java | 62 ++
.../samza/sql/runner/SamzaSqlApplication.java | 56 ++
.../sql/runner/SamzaSqlApplicationConfig.java | 245 +++++++
.../sql/runner/SamzaSqlApplicationRunner.java | 133 ++++
.../apache/samza/sql/testutil/ConfigUtil.java | 62 ++
.../org/apache/samza/sql/testutil/JsonUtil.java | 91 +++
.../samza/sql/testutil/ReflectionUtils.java | 62 ++
.../samza/sql/testutil/SamzaSqlQueryParser.java | 188 +++++
.../samza/sql/testutil/SqlFileParser.java | 103 +++
.../samza/sql/translator/FilterTranslator.java | 62 ++
.../samza/sql/translator/ProjectTranslator.java | 108 +++
.../samza/sql/translator/QueryTranslator.java | 96 +++
.../samza/sql/translator/ScanTranslator.java | 70 ++
.../samza/sql/translator/TranslatorContext.java | 162 +++++
.../apache/samza/sql/TestQueryTranslator.java | 103 +++
.../sql/TestSamzaSqlApplicationConfig.java | 92 +++
.../samza/sql/TestSamzaSqlFileParser.java | 58 ++
.../samza/sql/TestSamzaSqlQueryParser.java | 70 ++
.../samza/sql/TestSamzaSqlRelMessage.java | 46 ++
.../samza/sql/avro/TestAvroRelConversion.java | 239 +++++++
.../samza/sql/avro/schemas/ComplexRecord.avsc | 143 ++++
.../samza/sql/avro/schemas/ComplexRecord.java | 92 +++
.../apache/samza/sql/avro/schemas/MyFixed.java | 29 +
.../samza/sql/avro/schemas/SimpleRecord.avsc | 39 ++
.../samza/sql/avro/schemas/SimpleRecord.java | 52 ++
.../samza/sql/avro/schemas/SubRecord.java | 53 ++
.../samza/sql/avro/schemas/TestEnumType.java | 31 +
.../samza/sql/e2e/TestSamzaSqlEndToEnd.java | 137 ++++
.../sql/system/ConsoleLoggingSystemFactory.java | 83 +++
.../samza/sql/system/SimpleSystemAdmin.java | 61 ++
.../samza/sql/system/TestAvroSystemFactory.java | 156 +++++
.../samza/sql/testutil/MyTestArrayUdf.java | 37 +
.../apache/samza/sql/testutil/MyTestUdf.java | 45 ++
.../samza/sql/testutil/SamzaSqlTestConfig.java | 103 +++
samza-sql/src/test/resources/log4j.xml | 43 ++
.../performance/TestKeyValuePerformance.scala | 2 +-
.../samza/processor/TestZkStreamProcessor.java | 11 +-
.../processor/TestZkStreamProcessorBase.java | 3 +-
.../TestZkStreamProcessorFailures.java | 8 +-
.../processor/TestZkStreamProcessorSession.java | 5 +-
.../operator/TestRepartitionJoinWindowApp.java | 13 +-
.../processor/TestZkLocalApplicationRunner.java | 15 +-
settings.gradle | 5 +-
158 files changed, 8640 insertions(+), 1843 deletions(-)
----------------------------------------------------------------------