You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by pm...@apache.org on 2021/08/09 19:37:23 UTC
[samza] branch state-backend-async-commit updated (48699c4 ->
c95d828)
This is an automated email from the ASF dual-hosted git repository.
pmaheshwari pushed a change to branch state-backend-async-commit
in repository https://gitbox.apache.org/repos/asf/samza.git.
from 48699c4 Merge pull request #1512 from dxichen/checkpoint-restore-precendence-cleanup
new 8ecd445 SAMZA-2646: Minor refactor of StreamAppender to support sending formats other than byte[] to SystemProducer (#1488)
new 33bc5be SAMZA-2650: [Scala cleanup] Convert ShellCommandBuilder from scala to java (#1496)
new de067c4 SAMZA-2647: Clean up unused split deployment code (#1493)
new 8250a43 SAMZA-2652: Application Master High Availability metric - change counter to gauge (#1497)
new 2563caa SAMZA-2648: Add simple placeholder FaultDomainManager for when YARN is not used (#1494)
new 65e51ec SAMZA-2654: Allow coordinator url port to be configurable (#1499)
new 96b09b5 SAMZA-2649: Add MetricsReporter which logs metrics to log file (#1495)
new c2ef12f SAMZA-2653: Write pathing jar of run-class.sh into its own directory (#1498)
new 23627cc [SAMZA-2655] Expose RocksDB maxOpenFiles setting as Samza parameter (#1500)
new 9d9ebc7 SAMZA-398: Remove force NONE compression for changelog topic producer
new d5f5494 Merge pull request #1492 from perkss/SAMZA-398-allow-compressed-changelog
new 08931ce KafkaChangelogKeySerde should support deserializing multiple keys (#1502)
new 1f0f8bd SAMZA-1885: Enable parallel task execution for build (#1504)
new 337628e SAMZA-2660: Add authentication via TokenCredential for AzureBlobSystemProducer (#1507)
new 4c11067 SAMZA-2303: Exclude side inputs when handling end-of-stream and watermarks for high-level (#1506)
new 8fa07fe Samza-2665: AzureBlob SystemProducer: stop block upload retrying when InterruptedException is thrown (#1511)
new e975b86 Merge branch 'master' into state-backend-async-commit
new c95d828 Merge pull request #1513 from dxichen/merge-state-backend-async-commit
The 2376 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build.gradle | 7 +-
.../versioned/jobs/configuration-table.html | 16 +
.../versioned/jobs/samza-configurations.md | 11 +-
gradle.properties | 2 +
gradle/dependency-versions.gradle | 1 -
.../system/azureblob/AzureBlobClientBuilder.java | 148 +++++++++
.../samza/system/azureblob/AzureBlobConfig.java | 69 +++-
.../azureblob/avro/AzureBlobOutputStream.java | 7 +-
.../producer/AzureBlobSystemProducer.java | 80 ++---
.../azureblob/avro/TestAzureBlobOutputStream.java | 40 ++-
.../producer/TestAzureBlobSystemProducer.java | 16 +-
.../classloader/DependencyIsolationUtils.java | 50 ---
.../classloader/IsolatingClassLoaderFactory.java | 352 ---------------------
.../ClusterBasedJobCoordinatorRunner.java | 13 +-
.../clustermanager/SingleFaultDomainManager.java} | 41 +--
.../SingleFaultDomainManagerFactory.java} | 9 +-
.../apache/samza/config/ClusterManagerConfig.java | 18 ++
.../java/org/apache/samza/config/JobConfig.java | 6 -
.../apache/samza/config/ShellCommandConfig.java | 23 --
.../org/apache/samza/config/StorageConfig.java | 11 -
.../apache/samza/context/InternalTaskContext.java | 25 +-
.../org/apache/samza/context/TaskContextImpl.java | 27 +-
.../coordinator/JobCoordinatorMetadataManager.java | 35 +-
.../org/apache/samza/job/ShellCommandBuilder.java | 52 +++
.../metrics/reporter/LoggingMetricsReporter.java | 132 ++++++++
.../reporter/LoggingMetricsReporterConfig.java | 46 +++
.../reporter/LoggingMetricsReporterFactory.java | 45 +++
.../samza/operators/impl/OperatorImplGraph.java | 14 +-
.../samza/system/inmemory/InMemoryManager.java | 20 +-
.../org/apache/samza/util/SplitDeploymentUtil.java | 88 ------
.../org/apache/samza/container/TaskInstance.scala | 2 +-
.../apache/samza/coordinator/JobModelManager.scala | 3 +-
.../org/apache/samza/job/ShellCommandBuilder.scala | 54 ----
.../TestIsolatingClassLoaderFactory.java | 96 ------
.../org/apache/samza/config/TestJobConfig.java | 11 -
.../org/apache/samza/config/TestStorageConfig.java | 12 -
.../apache/samza/context/TestTaskContextImpl.java | 2 +-
.../TestJobCoordinatorMetadataManager.java | 10 +-
.../apache/samza/job/TestShellCommandBuilder.java | 90 ++++++
.../reporter/TestLoggingMetricsReporter.java | 198 ++++++++++++
.../reporter/TestLoggingMetricsReporterConfig.java | 54 ++++
.../samza/operators/impl/TestWindowOperator.java | 7 +-
.../samza/system/inmemory/TestInMemorySystem.java | 20 +-
.../apache/samza/util/TestSplitDeploymentUtil.java | 71 -----
.../apache/samza/job/TestShellCommandBuilder.scala | 61 ----
.../checkpoint/kafka/KafkaCheckpointLogKey.java | 4 -
.../kafka/KafkaCheckpointLogKeySerde.java | 3 +-
.../checkpoint/kafka/KafkaCheckpointManager.scala | 2 +-
.../org/apache/samza/config/KafkaConfig.scala | 4 +-
.../samza/system/kafka/KafkaSystemFactory.scala | 11 +-
.../kafka/TestKafkaCheckpointLogKeySerde.java | 16 +-
.../kafka/TestKafkaCheckpointManager.scala | 90 +++++-
.../system/kafka/TestKafkaSystemFactory.scala | 12 -
.../samza/storage/kv/RocksDbOptionsHelper.java | 4 +
.../kv/descriptors/RocksDbTableDescriptor.java | 40 +++
.../kv/descriptors/TestRocksDbTableDescriptor.java | 6 +-
.../samza/logging/log4j2/StreamAppender.java | 62 +++-
samza-shell/src/main/bash/run-class.sh | 45 ++-
.../apache/samza/test/framework/TestRunner.java | 8 +
.../StreamApplicationIntegrationTestHarness.java | 1 +
.../samza/test/table/TestLocalTableEndToEnd.java | 222 ++++++-------
.../TestLocalTableWithSideInputsEndToEnd.java | 188 +++++++----
.../org/apache/samza/test/table/TestTableData.java | 63 +++-
.../samza/job/yarn/SamzaAppMasterMetrics.scala | 4 +-
.../scala/org/apache/samza/job/yarn/YarnJob.scala | 28 +-
.../org/apache/samza/job/yarn/TestYarnJob.java | 45 ---
66 files changed, 1615 insertions(+), 1338 deletions(-)
create mode 100644 samza-azure/src/main/java/org/apache/samza/system/azureblob/AzureBlobClientBuilder.java
delete mode 100644 samza-core/src/main/java/org/apache/samza/classloader/DependencyIsolationUtils.java
delete mode 100644 samza-core/src/main/java/org/apache/samza/classloader/IsolatingClassLoaderFactory.java
copy samza-core/src/{test/java/org/apache/samza/clustermanager/MockClusterResourceManagerCallback.java => main/java/org/apache/samza/clustermanager/SingleFaultDomainManager.java} (50%)
copy samza-core/src/{test/java/org/apache/samza/clustermanager/MockFaultDomainManagerFactory.java => main/java/org/apache/samza/clustermanager/SingleFaultDomainManagerFactory.java} (81%)
create mode 100644 samza-core/src/main/java/org/apache/samza/job/ShellCommandBuilder.java
create mode 100644 samza-core/src/main/java/org/apache/samza/metrics/reporter/LoggingMetricsReporter.java
create mode 100644 samza-core/src/main/java/org/apache/samza/metrics/reporter/LoggingMetricsReporterConfig.java
create mode 100644 samza-core/src/main/java/org/apache/samza/metrics/reporter/LoggingMetricsReporterFactory.java
delete mode 100644 samza-core/src/main/java/org/apache/samza/util/SplitDeploymentUtil.java
delete mode 100644 samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala
delete mode 100644 samza-core/src/test/java/org/apache/samza/classloader/TestIsolatingClassLoaderFactory.java
create mode 100644 samza-core/src/test/java/org/apache/samza/job/TestShellCommandBuilder.java
create mode 100644 samza-core/src/test/java/org/apache/samza/metrics/reporter/TestLoggingMetricsReporter.java
create mode 100644 samza-core/src/test/java/org/apache/samza/metrics/reporter/TestLoggingMetricsReporterConfig.java
delete mode 100644 samza-core/src/test/java/org/apache/samza/util/TestSplitDeploymentUtil.java
delete mode 100644 samza-core/src/test/scala/org/apache/samza/job/TestShellCommandBuilder.scala