You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by pm...@apache.org on 2021/08/10 17:57:39 UTC
[samza] branch master updated (bad9ee8 -> f404029)
This is an automated email from the ASF dual-hosted git repository.
pmaheshwari pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git.
from bad9ee8 [SAMZA-2666] Updating tests to use TestRunner framework (#1510)
add aa58950 SAMZA-2591: Async Commit [1/3]: Checkpoint v2 migration (#1489)
add c117a68 SAMZA-2591: Async Commit [2/3]: Task Commit api changes and async commit (#1490)
add c85aade SAMZA-2591: Async Commit [3/3]: Container restore lifecycle (#1491)
add 7cc4eaa SAMZA-2657: Blob Store as backend for Samza State backup and restore (#1501)
add c2b77f0 Fix StorageConfig bug from newly introduced BlobStoreRestore factory config (#1503)
add 354620f SnapshotIndexSerde serialization fixes
add 48a8785 Checkpoint-tool changes for checkpoint v2
add 2063155 HOT FIX: KafkaConsumerProxy without any registered TopicPartitions
add 271ef64 Fix async cleanup TaskMetric
add 7d61a7d Added lastCommitNs and commitTimedOut metrics, updated default timeout to 30 mins
add 7f5ff3a Updated log line in BlobStoreUtil
add 35c0dbc Checkpoint version config precedence list
add 7676da9 Per store restore manager config precendence
add 5379f97 Job level state backend configs
add 0697457 Fixed case where there are no stores to backup for CSM
add e74c241 Added tests for CSM for non configured stores
add ad408d5 Cleanup of TODOs
add 785d506 checkstype cleanup
add 3c8a462 checkstyle for samza-api
add 65b5df6 Test fix for failing job: only backup persistent stores
add 48699c4 Merge pull request #1512 from dxichen/checkpoint-restore-precendence-cleanup
add e975b86 Merge branch 'master' into state-backend-async-commit
add c95d828 Merge pull request #1513 from dxichen/merge-state-backend-async-commit
new f404029 Merge pull request #1514 from apache/state-backend-async-commit
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build.gradle | 1 +
gradle/dependency-versions.gradle | 2 +-
.../org/apache/samza/checkpoint/Checkpoint.java | 57 +-
.../org/apache/samza/checkpoint/CheckpointId.java | 51 +-
.../{Checkpoint.java => CheckpointV1.java} | 37 +-
.../org/apache/samza/checkpoint/CheckpointV2.java | 123 +++
.../java/org/apache/samza/job/model/JobModel.java | 4 +
.../samza/storage/BlobStoreAdminFactory.java | 18 +-
.../samza/storage/KafkaChangelogRestoreParams.java | 77 ++
.../StateBackendAdmin.java} | 18 +-
.../apache/samza/storage/StateBackendFactory.java | 63 ++
.../org/apache/samza/storage/StorageEngine.java | 8 +
.../org/apache/samza/storage/StoreProperties.java | 23 +-
.../apache/samza/storage/TaskBackupManager.java | 92 ++
.../apache/samza/storage/TaskRestoreManager.java | 14 +-
.../samza/storage/blobstore/BlobStoreManager.java | 83 ++
.../storage/blobstore/BlobStoreManagerFactory.java | 13 +-
.../apache/samza/storage/blobstore/Metadata.java | 114 +++
.../blobstore/exceptions/DeletedException.java | 23 +-
.../blobstore/exceptions/RetriableException.java | 23 +-
.../org/apache/samza/storage/kv/KeyValueStore.java | 13 +
.../apache/samza/system/ChangelogSSPIterator.java | 2 +-
.../org/apache/samza/system/SystemFactory.java | 2 +-
.../apache/samza/checkpoint/TestCheckpointId.java | 53 ++
.../checkpoint/azure/AzureCheckpointManager.java | 10 +-
.../azure/ITestAzureCheckpointManager.java | 15 +-
.../checkpoint/kafka/KafkaChangelogSSPOffset.java | 40 +-
.../kafka/KafkaStateCheckpointMarker.java | 145 ++++
.../clustermanager/ClusterBasedJobCoordinator.java | 15 +
...ointManagerConfig.java => BlobStoreConfig.java} | 24 +-
.../java/org/apache/samza/config/JobConfig.java | 28 +
.../org/apache/samza/config/StorageConfig.java | 124 ++-
.../java/org/apache/samza/config/TaskConfig.java | 45 +
.../samza/coordinator/MetadataResourceUtil.java | 2 +-
.../samza/serializers/CheckpointV2Serde.java | 57 ++
.../apache/samza/serializers/JsonCheckpoint.java | 57 ++
...kModelMixIn.java => JsonCheckpointV2Mixin.java} | 31 +-
...n.java => KafkaStateCheckpointMarkerMixin.java} | 28 +-
.../samza/serializers/model/SamzaObjectMapper.java | 23 +
.../standalone/PassthroughJobCoordinator.java | 14 +-
.../apache/samza/startpoint/StartpointManager.java | 2 +-
.../storage/KafkaChangelogStateBackendFactory.java | 236 ++++++
.../NonTransactionalStateTaskRestoreManager.java | 81 +-
.../apache/samza/storage/StorageManagerUtil.java | 63 +-
.../org/apache/samza/storage/StorageRecovery.java | 16 +-
.../samza/storage/TaskRestoreManagerFactory.java | 82 --
.../samza/storage/TaskSideInputStorageManager.java | 18 +-
.../samza/storage/TaskStorageCommitManager.java | 366 ++++++++
.../TransactionalStateTaskRestoreManager.java | 130 ++-
.../storage/blobstore/BlobStoreBackupManager.java | 350 ++++++++
.../storage/blobstore/BlobStoreRestoreManager.java | 367 ++++++++
.../blobstore/BlobStoreStateBackendFactory.java | 100 +++
.../samza/storage/blobstore/diff/DirDiff.java | 244 ++++++
.../samza/storage/blobstore/index/DirIndex.java | 176 ++++
.../samza/storage/blobstore/index/FileBlob.java | 85 ++
.../samza/storage/blobstore/index/FileIndex.java | 114 +++
.../storage/blobstore/index/FileMetadata.java | 138 +++
.../storage/blobstore/index/SnapshotIndex.java | 124 +++
.../storage/blobstore/index/SnapshotMetadata.java | 115 +++
.../blobstore/index/serde/JsonDirIndexMixin.java} | 38 +-
.../blobstore/index/serde/JsonFileBlobMixin.java} | 19 +-
.../blobstore/index/serde/JsonFileIndexMixin.java} | 33 +-
.../index/serde/JsonFileMetadataMixin.java} | 37 +-
.../index/serde/JsonSnapshotIndexMixin.java} | 33 +-
.../index/serde/JsonSnapshotMetadataMixin.java} | 38 +-
.../blobstore/index/serde/SnapshotIndexSerde.java | 74 ++
.../metrics/BlobStoreBackupManagerMetrics.java | 136 +++
.../metrics/BlobStoreRestoreManagerMetrics.java | 94 +++
.../storage/blobstore/util/BlobStoreUtil.java | 622 ++++++++++++++
.../samza/storage/blobstore/util/DirDiffUtil.java | 403 +++++++++
.../java/org/apache/samza/system/SystemAdmins.java | 8 +
.../samza/system/inmemory/InMemoryManager.java | 2 +-
.../samza/table/retry/AsyncRetriableTable.java | 11 +-
.../apache/samza/table/retry/FailsafeAdapter.java | 43 +-
.../java/org/apache/samza/util/FutureUtil.java | 191 +++++
.../apache/samza/checkpoint/CheckpointTool.scala | 29 +-
.../apache/samza/checkpoint/OffsetManager.scala | 30 +-
.../file/FileSystemCheckpointManager.scala | 13 +-
.../apache/samza/container/SamzaContainer.scala | 161 ++--
.../org/apache/samza/container/TaskInstance.scala | 324 +++++--
.../samza/container/TaskInstanceMetrics.scala | 12 +-
.../apache/samza/job/local/ProcessJobFactory.scala | 15 +-
...eckpointSerde.scala => CheckpointV1Serde.scala} | 69 +-
.../samza/storage/ContainerStorageManager.java | 448 ++++++----
...fkaNonTransactionalStateTaskBackupManager.scala | 91 ++
.../KafkaTransactionalStateTaskBackupManager.scala | 97 +++
.../NonTransactionalStateTaskStorageManager.scala | 144 ----
.../apache/samza/storage/TaskStorageManager.scala | 37 -
.../samza/storage/TaskStorageManagerFactory.java | 46 -
.../TransactionalStateTaskStorageManager.scala | 201 -----
.../scala/org/apache/samza/util/FileUtil.scala | 9 +-
.../kafka/TestKafkaChangelogSSPOffset.java | 73 ++
.../kafka/TestKafkaStateCheckpointMarker.java | 114 +++
.../org/apache/samza/config/TestStorageConfig.java | 259 +++++-
.../org/apache/samza/job/model/TestJobModel.java | 2 +-
.../samza/serializers/TestCheckpointV2Serde.java | 96 +++
.../TestKafkaChangelogStateBackendFactory.java | 80 ++
.../storage/TestTaskStorageCommitManager.java | 916 ++++++++++++++++++++
.../TestTransactionalStateTaskRestoreManager.java | 252 +++---
.../blobstore/TestBlobStoreBackupManager.java | 542 ++++++++++++
.../blobstore/TestBlobStoreRestoreManager.java | 362 ++++++++
.../blobstore/serde/TestSnapshotIndexSerde.java | 56 ++
.../storage/blobstore/util/BlobStoreTestUtil.java | 296 +++++++
.../storage/blobstore/util/TestBlobStoreUtil.java | 936 +++++++++++++++++++++
.../storage/blobstore/util/TestDirDiffUtil.java | 127 +++
.../blobstore/util/TestDirDiffUtilMisc.java | 71 ++
.../java/org/apache/samza/util/TestFutureUtil.java | 222 +++++
.../samza/checkpoint/TestCheckpointTool.scala | 66 +-
.../samza/checkpoint/TestOffsetManager.scala | 30 +-
.../file/TestFileSystemCheckpointManager.scala | 10 +-
.../samza/container/TestSamzaContainer.scala | 34 -
.../apache/samza/container/TestTaskInstance.scala | 742 ++++++++++++++--
...ointSerde.scala => TestCheckpointV1Serde.scala} | 26 +-
.../samza/storage/TestContainerStorageManager.java | 266 +++++-
.../TestTransactionalStateTaskStorageManager.java | 534 ------------
.../scala/org/apache/samza/util/TestFileUtil.scala | 49 +-
.../checkpoint/kafka/KafkaCheckpointLogKey.java | 9 +-
.../kafka/KafkaCheckpointLogKeySerde.java | 7 +-
.../samza/system/kafka/KafkaConsumerProxy.java | 5 +-
.../checkpoint/kafka/KafkaCheckpointManager.scala | 123 ++-
.../kafka/TestKafkaCheckpointLogKeySerde.java | 14 +-
.../kafka/TestKafkaCheckpointManagerJava.java | 29 +-
.../samza/storage/TestTaskStorageManager.scala | 215 ++---
.../TestTransactionalStateTaskBackupManager.java | 276 ++++++
.../kafka/TestKafkaCheckpointManager.scala | 230 ++++-
.../InMemoryKeyValueStorageEngineFactory.java | 2 -
.../samza/storage/kv/RocksDbOptionsHelper.java | 124 ++-
.../kv/RocksDbKeyValueStorageEngineFactory.scala | 10 +-
.../samza/storage/kv/RocksDbKeyValueStore.scala | 19 +-
.../storage/kv/TestRocksDbKeyValueStore.scala | 20 +-
.../kv/BaseKeyValueStorageEngineFactory.java | 9 +-
.../kv/MockKeyValueStorageEngineFactory.java | 4 +-
.../kv/TestBaseKeyValueStorageEngineFactory.java | 37 +-
.../CheckpointVersionIntegrationTest.java | 149 ++++
.../samza/storage/MyStatefulApplication.java | 157 ++++
.../kv/TransactionalStateIntegrationTest.java | 135 +--
...ransactionalStateMultiStoreIntegrationTest.java | 120 +--
.../processor/TestZkLocalApplicationRunner.java | 4 +-
.../NonTransactionalStateIntegrationTest.scala | 5 +-
.../test/integration/StreamTaskTestUtil.scala | 8 +-
140 files changed, 13153 insertions(+), 2606 deletions(-)
copy samza-api/src/main/java/org/apache/samza/checkpoint/{Checkpoint.java => CheckpointV1.java} (69%)
create mode 100644 samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointV2.java
copy samza-core/src/main/java/org/apache/samza/container/disk/DiskQuotaPolicyFactory.java => samza-api/src/main/java/org/apache/samza/storage/BlobStoreAdminFactory.java (66%)
create mode 100644 samza-api/src/main/java/org/apache/samza/storage/KafkaChangelogRestoreParams.java
copy samza-api/src/main/java/org/apache/samza/{config/ConfigFactory.java => storage/StateBackendAdmin.java} (70%)
create mode 100644 samza-api/src/main/java/org/apache/samza/storage/StateBackendFactory.java
create mode 100644 samza-api/src/main/java/org/apache/samza/storage/TaskBackupManager.java
rename {samza-core => samza-api}/src/main/java/org/apache/samza/storage/TaskRestoreManager.java (86%)
create mode 100644 samza-api/src/main/java/org/apache/samza/storage/blobstore/BlobStoreManager.java
copy samza-core/src/main/java/org/apache/samza/clustermanager/FaultDomainManagerFactory.java => samza-api/src/main/java/org/apache/samza/storage/blobstore/BlobStoreManagerFactory.java (74%)
create mode 100644 samza-api/src/main/java/org/apache/samza/storage/blobstore/Metadata.java
copy samza-sql-shell/src/main/java/org/apache/samza/sql/client/exceptions/CliException.java => samza-api/src/main/java/org/apache/samza/storage/blobstore/exceptions/DeletedException.java (59%)
mode change 100755 => 100644
copy samza-sql-shell/src/main/java/org/apache/samza/sql/client/exceptions/CliException.java => samza-api/src/main/java/org/apache/samza/storage/blobstore/exceptions/RetriableException.java (71%)
mode change 100755 => 100644
create mode 100644 samza-api/src/test/java/org/apache/samza/checkpoint/TestCheckpointId.java
rename samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointedChangelogOffset.java => samza-core/src/main/java/org/apache/samza/checkpoint/kafka/KafkaChangelogSSPOffset.java (57%)
create mode 100644 samza-core/src/main/java/org/apache/samza/checkpoint/kafka/KafkaStateCheckpointMarker.java
copy samza-core/src/main/java/org/apache/samza/config/{FileSystemCheckpointManagerConfig.java => BlobStoreConfig.java} (63%)
create mode 100644 samza-core/src/main/java/org/apache/samza/serializers/CheckpointV2Serde.java
create mode 100644 samza-core/src/main/java/org/apache/samza/serializers/JsonCheckpoint.java
copy samza-core/src/main/java/org/apache/samza/serializers/model/{JsonTaskModelMixIn.java => JsonCheckpointV2Mixin.java} (61%)
copy samza-core/src/main/java/org/apache/samza/serializers/model/{JsonProcessorLocalityMixIn.java => KafkaStateCheckpointMarkerMixin.java} (61%)
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/KafkaChangelogStateBackendFactory.java
delete mode 100644 samza-core/src/main/java/org/apache/samza/storage/TaskRestoreManagerFactory.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/TaskStorageCommitManager.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreBackupManager.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreRestoreManager.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreStateBackendFactory.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/diff/DirDiff.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/DirIndex.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileBlob.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileIndex.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileMetadata.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/SnapshotIndex.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/SnapshotMetadata.java
copy samza-core/src/main/java/org/apache/samza/{serializers/model/JsonProcessorLocalityMixIn.java => storage/blobstore/index/serde/JsonDirIndexMixin.java} (50%)
copy samza-core/src/main/java/org/apache/samza/{serializers/model/JsonLocalityModelMixIn.java => storage/blobstore/index/serde/JsonFileBlobMixin.java} (68%)
copy samza-core/src/main/java/org/apache/samza/{serializers/model/JsonProcessorLocalityMixIn.java => storage/blobstore/index/serde/JsonFileIndexMixin.java} (57%)
copy samza-core/src/main/java/org/apache/samza/{serializers/model/JsonProcessorLocalityMixIn.java => storage/blobstore/index/serde/JsonFileMetadataMixin.java} (53%)
copy samza-core/src/main/java/org/apache/samza/{serializers/model/JsonProcessorLocalityMixIn.java => storage/blobstore/index/serde/JsonSnapshotIndexMixin.java} (51%)
copy samza-core/src/main/java/org/apache/samza/{serializers/model/JsonTaskModelMixIn.java => storage/blobstore/index/serde/JsonSnapshotMetadataMixin.java} (57%)
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/SnapshotIndexSerde.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/metrics/BlobStoreBackupManagerMetrics.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/metrics/BlobStoreRestoreManagerMetrics.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/util/BlobStoreUtil.java
create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/util/DirDiffUtil.java
create mode 100644 samza-core/src/main/java/org/apache/samza/util/FutureUtil.java
rename samza-core/src/main/scala/org/apache/samza/serializers/{CheckpointSerde.scala => CheckpointV1Serde.scala} (54%)
create mode 100644 samza-core/src/main/scala/org/apache/samza/storage/KafkaNonTransactionalStateTaskBackupManager.scala
create mode 100644 samza-core/src/main/scala/org/apache/samza/storage/KafkaTransactionalStateTaskBackupManager.scala
delete mode 100644 samza-core/src/main/scala/org/apache/samza/storage/NonTransactionalStateTaskStorageManager.scala
delete mode 100644 samza-core/src/main/scala/org/apache/samza/storage/TaskStorageManager.scala
delete mode 100644 samza-core/src/main/scala/org/apache/samza/storage/TaskStorageManagerFactory.java
delete mode 100644 samza-core/src/main/scala/org/apache/samza/storage/TransactionalStateTaskStorageManager.scala
create mode 100644 samza-core/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaChangelogSSPOffset.java
create mode 100644 samza-core/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaStateCheckpointMarker.java
create mode 100644 samza-core/src/test/java/org/apache/samza/serializers/TestCheckpointV2Serde.java
create mode 100644 samza-core/src/test/java/org/apache/samza/storage/TestKafkaChangelogStateBackendFactory.java
create mode 100644 samza-core/src/test/java/org/apache/samza/storage/TestTaskStorageCommitManager.java
create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/TestBlobStoreBackupManager.java
create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/TestBlobStoreRestoreManager.java
create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/serde/TestSnapshotIndexSerde.java
create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/util/BlobStoreTestUtil.java
create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestBlobStoreUtil.java
create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestDirDiffUtil.java
create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestDirDiffUtilMisc.java
create mode 100644 samza-core/src/test/java/org/apache/samza/util/TestFutureUtil.java
rename samza-core/src/test/scala/org/apache/samza/serializers/{TestCheckpointSerde.scala => TestCheckpointV1Serde.scala} (69%)
delete mode 100644 samza-core/src/test/scala/org/apache/samza/storage/TestTransactionalStateTaskStorageManager.java
rename {samza-core/src/test/scala => samza-kafka/src/test/java}/org/apache/samza/storage/TestTaskStorageManager.scala (83%)
create mode 100644 samza-kafka/src/test/java/org/apache/samza/storage/TestTransactionalStateTaskBackupManager.java
create mode 100644 samza-test/src/test/java/org/apache/samza/checkpoint/CheckpointVersionIntegrationTest.java
create mode 100644 samza-test/src/test/java/org/apache/samza/storage/MyStatefulApplication.java
[samza] 01/01: Merge pull request #1514 from
apache/state-backend-async-commit
Posted by pm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pmaheshwari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git
commit f40402948f37d84c000f1b4340180e538ac75c82
Merge: bad9ee8 c95d828
Author: Prateek Maheshwari <pr...@utexas.edu>
AuthorDate: Tue Aug 10 10:57:32 2021 -0700
Merge pull request #1514 from apache/state-backend-async-commit
All changes for
SEP-28: https://cwiki.apache.org/confluence/display/SAMZA/SEP-28%3A+Samza+State+Backend+Interface+and+Checkpointing+Improvements
SEP-29: https://cwiki.apache.org/confluence/display/SAMZA/SEP-29%3A+Blob+Store+Based+State+Backup+And+Restore
build.gradle | 1 +
gradle/dependency-versions.gradle | 2 +-
.../org/apache/samza/checkpoint/Checkpoint.java | 57 +-
.../org/apache/samza/checkpoint/CheckpointId.java | 51 +-
.../{Checkpoint.java => CheckpointV1.java} | 37 +-
.../org/apache/samza/checkpoint/CheckpointV2.java | 123 +++
.../java/org/apache/samza/job/model/JobModel.java | 4 +
.../samza/storage/BlobStoreAdminFactory.java | 30 +-
.../samza/storage/KafkaChangelogRestoreParams.java | 77 ++
.../apache/samza/storage/StateBackendAdmin.java | 32 +-
.../apache/samza/storage/StateBackendFactory.java | 63 ++
.../org/apache/samza/storage/StorageEngine.java | 8 +
.../org/apache/samza/storage/StoreProperties.java | 23 +-
.../apache/samza/storage/TaskBackupManager.java | 92 ++
.../apache/samza/storage/TaskRestoreManager.java | 14 +-
.../samza/storage/blobstore/BlobStoreManager.java | 83 ++
.../storage/blobstore/BlobStoreManagerFactory.java | 24 +-
.../apache/samza/storage/blobstore/Metadata.java | 114 +++
.../blobstore/exceptions/DeletedException.java | 33 +-
.../blobstore/exceptions/RetriableException.java | 25 +-
.../org/apache/samza/storage/kv/KeyValueStore.java | 13 +
.../apache/samza/system/ChangelogSSPIterator.java | 2 +-
.../org/apache/samza/system/SystemFactory.java | 2 +-
.../apache/samza/checkpoint/TestCheckpointId.java | 53 ++
.../checkpoint/azure/AzureCheckpointManager.java | 10 +-
.../azure/ITestAzureCheckpointManager.java | 15 +-
.../checkpoint/kafka/KafkaChangelogSSPOffset.java | 40 +-
.../kafka/KafkaStateCheckpointMarker.java | 145 ++++
.../clustermanager/ClusterBasedJobCoordinator.java | 15 +
.../org/apache/samza/config/BlobStoreConfig.java} | 30 +-
.../java/org/apache/samza/config/JobConfig.java | 28 +
.../org/apache/samza/config/StorageConfig.java | 124 ++-
.../java/org/apache/samza/config/TaskConfig.java | 45 +
.../samza/coordinator/MetadataResourceUtil.java | 2 +-
.../samza/serializers/CheckpointV2Serde.java | 57 ++
.../apache/samza/serializers/JsonCheckpoint.java | 57 ++
.../serializers/model/JsonCheckpointV2Mixin.java | 46 +
.../model/KafkaStateCheckpointMarkerMixin.java | 48 ++
.../samza/serializers/model/SamzaObjectMapper.java | 23 +
.../standalone/PassthroughJobCoordinator.java | 14 +-
.../apache/samza/startpoint/StartpointManager.java | 2 +-
.../storage/KafkaChangelogStateBackendFactory.java | 236 ++++++
.../NonTransactionalStateTaskRestoreManager.java | 81 +-
.../apache/samza/storage/StorageManagerUtil.java | 63 +-
.../org/apache/samza/storage/StorageRecovery.java | 16 +-
.../samza/storage/TaskRestoreManagerFactory.java | 82 --
.../samza/storage/TaskSideInputStorageManager.java | 18 +-
.../samza/storage/TaskStorageCommitManager.java | 366 ++++++++
.../TransactionalStateTaskRestoreManager.java | 130 ++-
.../storage/blobstore/BlobStoreBackupManager.java | 350 ++++++++
.../storage/blobstore/BlobStoreRestoreManager.java | 367 ++++++++
.../blobstore/BlobStoreStateBackendFactory.java | 100 +++
.../samza/storage/blobstore/diff/DirDiff.java | 244 ++++++
.../samza/storage/blobstore/index/DirIndex.java | 176 ++++
.../samza/storage/blobstore/index/FileBlob.java | 85 ++
.../samza/storage/blobstore/index/FileIndex.java | 114 +++
.../storage/blobstore/index/FileMetadata.java | 138 +++
.../storage/blobstore/index/SnapshotIndex.java | 124 +++
.../storage/blobstore/index/SnapshotMetadata.java | 115 +++
.../blobstore/index/serde/JsonDirIndexMixin.java | 58 ++
.../blobstore/index/serde/JsonFileBlobMixin.java} | 34 +-
.../blobstore/index/serde/JsonFileIndexMixin.java | 53 ++
.../index/serde/JsonFileMetadataMixin.java | 55 ++
.../index/serde/JsonSnapshotIndexMixin.java | 53 ++
.../index/serde/JsonSnapshotMetadataMixin.java | 53 ++
.../blobstore/index/serde/SnapshotIndexSerde.java | 74 ++
.../metrics/BlobStoreBackupManagerMetrics.java | 136 +++
.../metrics/BlobStoreRestoreManagerMetrics.java | 94 +++
.../storage/blobstore/util/BlobStoreUtil.java | 622 ++++++++++++++
.../samza/storage/blobstore/util/DirDiffUtil.java | 403 +++++++++
.../java/org/apache/samza/system/SystemAdmins.java | 8 +
.../samza/system/inmemory/InMemoryManager.java | 2 +-
.../samza/table/retry/AsyncRetriableTable.java | 11 +-
.../apache/samza/table/retry/FailsafeAdapter.java | 43 +-
.../java/org/apache/samza/util/FutureUtil.java | 191 +++++
.../apache/samza/checkpoint/CheckpointTool.scala | 29 +-
.../apache/samza/checkpoint/OffsetManager.scala | 30 +-
.../file/FileSystemCheckpointManager.scala | 13 +-
.../apache/samza/container/SamzaContainer.scala | 161 ++--
.../org/apache/samza/container/TaskInstance.scala | 324 +++++--
.../samza/container/TaskInstanceMetrics.scala | 12 +-
.../apache/samza/job/local/ProcessJobFactory.scala | 15 +-
...eckpointSerde.scala => CheckpointV1Serde.scala} | 69 +-
.../samza/storage/ContainerStorageManager.java | 448 ++++++----
...fkaNonTransactionalStateTaskBackupManager.scala | 91 ++
.../KafkaTransactionalStateTaskBackupManager.scala | 97 +++
.../NonTransactionalStateTaskStorageManager.scala | 144 ----
.../samza/storage/TaskStorageManagerFactory.java | 46 -
.../TransactionalStateTaskStorageManager.scala | 201 -----
.../scala/org/apache/samza/util/FileUtil.scala | 9 +-
.../kafka/TestKafkaChangelogSSPOffset.java | 73 ++
.../kafka/TestKafkaStateCheckpointMarker.java | 114 +++
.../org/apache/samza/config/TestStorageConfig.java | 259 +++++-
.../org/apache/samza/job/model/TestJobModel.java | 2 +-
.../samza/serializers/TestCheckpointV2Serde.java | 96 +++
.../TestKafkaChangelogStateBackendFactory.java | 80 ++
.../storage/TestTaskStorageCommitManager.java | 916 ++++++++++++++++++++
.../TestTransactionalStateTaskRestoreManager.java | 252 +++---
.../blobstore/TestBlobStoreBackupManager.java | 542 ++++++++++++
.../blobstore/TestBlobStoreRestoreManager.java | 362 ++++++++
.../blobstore/serde/TestSnapshotIndexSerde.java | 56 ++
.../storage/blobstore/util/BlobStoreTestUtil.java | 296 +++++++
.../storage/blobstore/util/TestBlobStoreUtil.java | 936 +++++++++++++++++++++
.../storage/blobstore/util/TestDirDiffUtil.java | 127 +++
.../blobstore/util/TestDirDiffUtilMisc.java | 71 ++
.../java/org/apache/samza/util/TestFutureUtil.java | 222 +++++
.../samza/checkpoint/TestCheckpointTool.scala | 66 +-
.../samza/checkpoint/TestOffsetManager.scala | 30 +-
.../file/TestFileSystemCheckpointManager.scala | 10 +-
.../samza/container/TestSamzaContainer.scala | 34 -
.../apache/samza/container/TestTaskInstance.scala | 742 ++++++++++++++--
...ointSerde.scala => TestCheckpointV1Serde.scala} | 26 +-
.../samza/storage/TestContainerStorageManager.java | 266 +++++-
.../TestTransactionalStateTaskStorageManager.java | 534 ------------
.../scala/org/apache/samza/util/TestFileUtil.scala | 49 +-
.../checkpoint/kafka/KafkaCheckpointLogKey.java | 9 +-
.../kafka/KafkaCheckpointLogKeySerde.java | 7 +-
.../samza/system/kafka/KafkaConsumerProxy.java | 5 +-
.../checkpoint/kafka/KafkaCheckpointManager.scala | 123 ++-
.../kafka/TestKafkaCheckpointLogKeySerde.java | 14 +-
.../kafka/TestKafkaCheckpointManagerJava.java | 29 +-
.../samza/storage/TestTaskStorageManager.scala | 215 ++---
.../TestTransactionalStateTaskBackupManager.java | 276 ++++++
.../kafka/TestKafkaCheckpointManager.scala | 230 ++++-
.../InMemoryKeyValueStorageEngineFactory.java | 2 -
.../samza/storage/kv/RocksDbOptionsHelper.java | 124 ++-
.../kv/RocksDbKeyValueStorageEngineFactory.scala | 10 +-
.../samza/storage/kv/RocksDbKeyValueStore.scala | 19 +-
.../storage/kv/TestRocksDbKeyValueStore.scala | 20 +-
.../kv/BaseKeyValueStorageEngineFactory.java | 9 +-
.../kv/MockKeyValueStorageEngineFactory.java | 4 +-
.../kv/TestBaseKeyValueStorageEngineFactory.java | 37 +-
.../CheckpointVersionIntegrationTest.java | 149 ++++
.../samza/storage/MyStatefulApplication.java | 157 ++++
.../kv/TransactionalStateIntegrationTest.java | 135 +--
...ransactionalStateMultiStoreIntegrationTest.java | 120 +--
.../processor/TestZkLocalApplicationRunner.java | 4 +-
.../NonTransactionalStateIntegrationTest.scala | 5 +-
.../test/integration/StreamTaskTestUtil.scala | 8 +-
139 files changed, 13422 insertions(+), 2498 deletions(-)