You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by pm...@apache.org on 2021/08/10 17:57:39 UTC

[samza] branch master updated (bad9ee8 -> f404029)

This is an automated email from the ASF dual-hosted git repository.

pmaheshwari pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git.


    from bad9ee8  [SAMZA-2666] Updating tests to use TestRunner framework (#1510)
     add aa58950  SAMZA-2591: Async Commit [1/3]: Checkpoint v2 migration (#1489)
     add c117a68  SAMZA-2591: Async Commit [2/3]: Task Commit api changes and async commit (#1490)
     add c85aade  SAMZA-2591: Async Commit [3/3]: Container restore lifecycle (#1491)
     add 7cc4eaa  SAMZA-2657: Blob Store as backend for Samza State backup and restore (#1501)
     add c2b77f0  Fix StorageConfig bug from newly introduced BlobStoreRestore factory config (#1503)
     add 354620f  SnapshotIndexSerde serialization fixes
     add 48a8785  Checkpoint-tool changes for checkpoint v2
     add 2063155  HOT FIX: KafkaConsumerProxy without any registered TopicPartitions
     add 271ef64  Fix async cleanup TaskMetric
     add 7d61a7d  Added lastCommitNs and commitTimedOut metrics, updated default timeout to 30 mins
     add 7f5ff3a  Updated log line in BlobStoreUtil
     add 35c0dbc  Checkpoint version config precedence list
     add 7676da9  Per store restore manager config precendence
     add 5379f97  Job level state backend configs
     add 0697457  Fixed case where there are no stores to backup for CSM
     add e74c241  Added tests for CSM for non configured stores
     add ad408d5  Cleanup of TODOs
     add 785d506  checkstype cleanup
     add 3c8a462  checkstyle for samza-api
     add 65b5df6  Test fix for failing job: only backup persistent stores
     add 48699c4  Merge pull request #1512 from dxichen/checkpoint-restore-precendence-cleanup
     add e975b86  Merge branch 'master' into state-backend-async-commit
     add c95d828  Merge pull request #1513 from dxichen/merge-state-backend-async-commit
     new f404029  Merge pull request #1514 from apache/state-backend-async-commit

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 build.gradle                                       |   1 +
 gradle/dependency-versions.gradle                  |   2 +-
 .../org/apache/samza/checkpoint/Checkpoint.java    |  57 +-
 .../org/apache/samza/checkpoint/CheckpointId.java  |  51 +-
 .../{Checkpoint.java => CheckpointV1.java}         |  37 +-
 .../org/apache/samza/checkpoint/CheckpointV2.java  | 123 +++
 .../java/org/apache/samza/job/model/JobModel.java  |   4 +
 .../samza/storage/BlobStoreAdminFactory.java       |  18 +-
 .../samza/storage/KafkaChangelogRestoreParams.java |  77 ++
 .../StateBackendAdmin.java}                        |  18 +-
 .../apache/samza/storage/StateBackendFactory.java  |  63 ++
 .../org/apache/samza/storage/StorageEngine.java    |   8 +
 .../org/apache/samza/storage/StoreProperties.java  |  23 +-
 .../apache/samza/storage/TaskBackupManager.java    |  92 ++
 .../apache/samza/storage/TaskRestoreManager.java   |  14 +-
 .../samza/storage/blobstore/BlobStoreManager.java  |  83 ++
 .../storage/blobstore/BlobStoreManagerFactory.java |  13 +-
 .../apache/samza/storage/blobstore/Metadata.java   | 114 +++
 .../blobstore/exceptions/DeletedException.java     |  23 +-
 .../blobstore/exceptions/RetriableException.java   |  23 +-
 .../org/apache/samza/storage/kv/KeyValueStore.java |  13 +
 .../apache/samza/system/ChangelogSSPIterator.java  |   2 +-
 .../org/apache/samza/system/SystemFactory.java     |   2 +-
 .../apache/samza/checkpoint/TestCheckpointId.java  |  53 ++
 .../checkpoint/azure/AzureCheckpointManager.java   |  10 +-
 .../azure/ITestAzureCheckpointManager.java         |  15 +-
 .../checkpoint/kafka/KafkaChangelogSSPOffset.java  |  40 +-
 .../kafka/KafkaStateCheckpointMarker.java          | 145 ++++
 .../clustermanager/ClusterBasedJobCoordinator.java |  15 +
 ...ointManagerConfig.java => BlobStoreConfig.java} |  24 +-
 .../java/org/apache/samza/config/JobConfig.java    |  28 +
 .../org/apache/samza/config/StorageConfig.java     | 124 ++-
 .../java/org/apache/samza/config/TaskConfig.java   |  45 +
 .../samza/coordinator/MetadataResourceUtil.java    |   2 +-
 .../samza/serializers/CheckpointV2Serde.java       |  57 ++
 .../apache/samza/serializers/JsonCheckpoint.java   |  57 ++
 ...kModelMixIn.java => JsonCheckpointV2Mixin.java} |  31 +-
 ...n.java => KafkaStateCheckpointMarkerMixin.java} |  28 +-
 .../samza/serializers/model/SamzaObjectMapper.java |  23 +
 .../standalone/PassthroughJobCoordinator.java      |  14 +-
 .../apache/samza/startpoint/StartpointManager.java |   2 +-
 .../storage/KafkaChangelogStateBackendFactory.java | 236 ++++++
 .../NonTransactionalStateTaskRestoreManager.java   |  81 +-
 .../apache/samza/storage/StorageManagerUtil.java   |  63 +-
 .../org/apache/samza/storage/StorageRecovery.java  |  16 +-
 .../samza/storage/TaskRestoreManagerFactory.java   |  82 --
 .../samza/storage/TaskSideInputStorageManager.java |  18 +-
 .../samza/storage/TaskStorageCommitManager.java    | 366 ++++++++
 .../TransactionalStateTaskRestoreManager.java      | 130 ++-
 .../storage/blobstore/BlobStoreBackupManager.java  | 350 ++++++++
 .../storage/blobstore/BlobStoreRestoreManager.java | 367 ++++++++
 .../blobstore/BlobStoreStateBackendFactory.java    | 100 +++
 .../samza/storage/blobstore/diff/DirDiff.java      | 244 ++++++
 .../samza/storage/blobstore/index/DirIndex.java    | 176 ++++
 .../samza/storage/blobstore/index/FileBlob.java    |  85 ++
 .../samza/storage/blobstore/index/FileIndex.java   | 114 +++
 .../storage/blobstore/index/FileMetadata.java      | 138 +++
 .../storage/blobstore/index/SnapshotIndex.java     | 124 +++
 .../storage/blobstore/index/SnapshotMetadata.java  | 115 +++
 .../blobstore/index/serde/JsonDirIndexMixin.java}  |  38 +-
 .../blobstore/index/serde/JsonFileBlobMixin.java}  |  19 +-
 .../blobstore/index/serde/JsonFileIndexMixin.java} |  33 +-
 .../index/serde/JsonFileMetadataMixin.java}        |  37 +-
 .../index/serde/JsonSnapshotIndexMixin.java}       |  33 +-
 .../index/serde/JsonSnapshotMetadataMixin.java}    |  38 +-
 .../blobstore/index/serde/SnapshotIndexSerde.java  |  74 ++
 .../metrics/BlobStoreBackupManagerMetrics.java     | 136 +++
 .../metrics/BlobStoreRestoreManagerMetrics.java    |  94 +++
 .../storage/blobstore/util/BlobStoreUtil.java      | 622 ++++++++++++++
 .../samza/storage/blobstore/util/DirDiffUtil.java  | 403 +++++++++
 .../java/org/apache/samza/system/SystemAdmins.java |   8 +
 .../samza/system/inmemory/InMemoryManager.java     |   2 +-
 .../samza/table/retry/AsyncRetriableTable.java     |  11 +-
 .../apache/samza/table/retry/FailsafeAdapter.java  |  43 +-
 .../java/org/apache/samza/util/FutureUtil.java     | 191 +++++
 .../apache/samza/checkpoint/CheckpointTool.scala   |  29 +-
 .../apache/samza/checkpoint/OffsetManager.scala    |  30 +-
 .../file/FileSystemCheckpointManager.scala         |  13 +-
 .../apache/samza/container/SamzaContainer.scala    | 161 ++--
 .../org/apache/samza/container/TaskInstance.scala  | 324 +++++--
 .../samza/container/TaskInstanceMetrics.scala      |  12 +-
 .../apache/samza/job/local/ProcessJobFactory.scala |  15 +-
 ...eckpointSerde.scala => CheckpointV1Serde.scala} |  69 +-
 .../samza/storage/ContainerStorageManager.java     | 448 ++++++----
 ...fkaNonTransactionalStateTaskBackupManager.scala |  91 ++
 .../KafkaTransactionalStateTaskBackupManager.scala |  97 +++
 .../NonTransactionalStateTaskStorageManager.scala  | 144 ----
 .../apache/samza/storage/TaskStorageManager.scala  |  37 -
 .../samza/storage/TaskStorageManagerFactory.java   |  46 -
 .../TransactionalStateTaskStorageManager.scala     | 201 -----
 .../scala/org/apache/samza/util/FileUtil.scala     |   9 +-
 .../kafka/TestKafkaChangelogSSPOffset.java         |  73 ++
 .../kafka/TestKafkaStateCheckpointMarker.java      | 114 +++
 .../org/apache/samza/config/TestStorageConfig.java | 259 +++++-
 .../org/apache/samza/job/model/TestJobModel.java   |   2 +-
 .../samza/serializers/TestCheckpointV2Serde.java   |  96 +++
 .../TestKafkaChangelogStateBackendFactory.java     |  80 ++
 .../storage/TestTaskStorageCommitManager.java      | 916 ++++++++++++++++++++
 .../TestTransactionalStateTaskRestoreManager.java  | 252 +++---
 .../blobstore/TestBlobStoreBackupManager.java      | 542 ++++++++++++
 .../blobstore/TestBlobStoreRestoreManager.java     | 362 ++++++++
 .../blobstore/serde/TestSnapshotIndexSerde.java    |  56 ++
 .../storage/blobstore/util/BlobStoreTestUtil.java  | 296 +++++++
 .../storage/blobstore/util/TestBlobStoreUtil.java  | 936 +++++++++++++++++++++
 .../storage/blobstore/util/TestDirDiffUtil.java    | 127 +++
 .../blobstore/util/TestDirDiffUtilMisc.java        |  71 ++
 .../java/org/apache/samza/util/TestFutureUtil.java | 222 +++++
 .../samza/checkpoint/TestCheckpointTool.scala      |  66 +-
 .../samza/checkpoint/TestOffsetManager.scala       |  30 +-
 .../file/TestFileSystemCheckpointManager.scala     |  10 +-
 .../samza/container/TestSamzaContainer.scala       |  34 -
 .../apache/samza/container/TestTaskInstance.scala  | 742 ++++++++++++++--
 ...ointSerde.scala => TestCheckpointV1Serde.scala} |  26 +-
 .../samza/storage/TestContainerStorageManager.java | 266 +++++-
 .../TestTransactionalStateTaskStorageManager.java  | 534 ------------
 .../scala/org/apache/samza/util/TestFileUtil.scala |  49 +-
 .../checkpoint/kafka/KafkaCheckpointLogKey.java    |   9 +-
 .../kafka/KafkaCheckpointLogKeySerde.java          |   7 +-
 .../samza/system/kafka/KafkaConsumerProxy.java     |   5 +-
 .../checkpoint/kafka/KafkaCheckpointManager.scala  | 123 ++-
 .../kafka/TestKafkaCheckpointLogKeySerde.java      |  14 +-
 .../kafka/TestKafkaCheckpointManagerJava.java      |  29 +-
 .../samza/storage/TestTaskStorageManager.scala     | 215 ++---
 .../TestTransactionalStateTaskBackupManager.java   | 276 ++++++
 .../kafka/TestKafkaCheckpointManager.scala         | 230 ++++-
 .../InMemoryKeyValueStorageEngineFactory.java      |   2 -
 .../samza/storage/kv/RocksDbOptionsHelper.java     | 124 ++-
 .../kv/RocksDbKeyValueStorageEngineFactory.scala   |  10 +-
 .../samza/storage/kv/RocksDbKeyValueStore.scala    |  19 +-
 .../storage/kv/TestRocksDbKeyValueStore.scala      |  20 +-
 .../kv/BaseKeyValueStorageEngineFactory.java       |   9 +-
 .../kv/MockKeyValueStorageEngineFactory.java       |   4 +-
 .../kv/TestBaseKeyValueStorageEngineFactory.java   |  37 +-
 .../CheckpointVersionIntegrationTest.java          | 149 ++++
 .../samza/storage/MyStatefulApplication.java       | 157 ++++
 .../kv/TransactionalStateIntegrationTest.java      | 135 +--
 ...ransactionalStateMultiStoreIntegrationTest.java | 120 +--
 .../processor/TestZkLocalApplicationRunner.java    |   4 +-
 .../NonTransactionalStateIntegrationTest.scala     |   5 +-
 .../test/integration/StreamTaskTestUtil.scala      |   8 +-
 140 files changed, 13153 insertions(+), 2606 deletions(-)
 copy samza-api/src/main/java/org/apache/samza/checkpoint/{Checkpoint.java => CheckpointV1.java} (69%)
 create mode 100644 samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointV2.java
 copy samza-core/src/main/java/org/apache/samza/container/disk/DiskQuotaPolicyFactory.java => samza-api/src/main/java/org/apache/samza/storage/BlobStoreAdminFactory.java (66%)
 create mode 100644 samza-api/src/main/java/org/apache/samza/storage/KafkaChangelogRestoreParams.java
 copy samza-api/src/main/java/org/apache/samza/{config/ConfigFactory.java => storage/StateBackendAdmin.java} (70%)
 create mode 100644 samza-api/src/main/java/org/apache/samza/storage/StateBackendFactory.java
 create mode 100644 samza-api/src/main/java/org/apache/samza/storage/TaskBackupManager.java
 rename {samza-core => samza-api}/src/main/java/org/apache/samza/storage/TaskRestoreManager.java (86%)
 create mode 100644 samza-api/src/main/java/org/apache/samza/storage/blobstore/BlobStoreManager.java
 copy samza-core/src/main/java/org/apache/samza/clustermanager/FaultDomainManagerFactory.java => samza-api/src/main/java/org/apache/samza/storage/blobstore/BlobStoreManagerFactory.java (74%)
 create mode 100644 samza-api/src/main/java/org/apache/samza/storage/blobstore/Metadata.java
 copy samza-sql-shell/src/main/java/org/apache/samza/sql/client/exceptions/CliException.java => samza-api/src/main/java/org/apache/samza/storage/blobstore/exceptions/DeletedException.java (59%)
 mode change 100755 => 100644
 copy samza-sql-shell/src/main/java/org/apache/samza/sql/client/exceptions/CliException.java => samza-api/src/main/java/org/apache/samza/storage/blobstore/exceptions/RetriableException.java (71%)
 mode change 100755 => 100644
 create mode 100644 samza-api/src/test/java/org/apache/samza/checkpoint/TestCheckpointId.java
 rename samza-api/src/main/java/org/apache/samza/checkpoint/CheckpointedChangelogOffset.java => samza-core/src/main/java/org/apache/samza/checkpoint/kafka/KafkaChangelogSSPOffset.java (57%)
 create mode 100644 samza-core/src/main/java/org/apache/samza/checkpoint/kafka/KafkaStateCheckpointMarker.java
 copy samza-core/src/main/java/org/apache/samza/config/{FileSystemCheckpointManagerConfig.java => BlobStoreConfig.java} (63%)
 create mode 100644 samza-core/src/main/java/org/apache/samza/serializers/CheckpointV2Serde.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/serializers/JsonCheckpoint.java
 copy samza-core/src/main/java/org/apache/samza/serializers/model/{JsonTaskModelMixIn.java => JsonCheckpointV2Mixin.java} (61%)
 copy samza-core/src/main/java/org/apache/samza/serializers/model/{JsonProcessorLocalityMixIn.java => KafkaStateCheckpointMarkerMixin.java} (61%)
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/KafkaChangelogStateBackendFactory.java
 delete mode 100644 samza-core/src/main/java/org/apache/samza/storage/TaskRestoreManagerFactory.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/TaskStorageCommitManager.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreBackupManager.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreRestoreManager.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/BlobStoreStateBackendFactory.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/diff/DirDiff.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/DirIndex.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileBlob.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileIndex.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/FileMetadata.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/SnapshotIndex.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/SnapshotMetadata.java
 copy samza-core/src/main/java/org/apache/samza/{serializers/model/JsonProcessorLocalityMixIn.java => storage/blobstore/index/serde/JsonDirIndexMixin.java} (50%)
 copy samza-core/src/main/java/org/apache/samza/{serializers/model/JsonLocalityModelMixIn.java => storage/blobstore/index/serde/JsonFileBlobMixin.java} (68%)
 copy samza-core/src/main/java/org/apache/samza/{serializers/model/JsonProcessorLocalityMixIn.java => storage/blobstore/index/serde/JsonFileIndexMixin.java} (57%)
 copy samza-core/src/main/java/org/apache/samza/{serializers/model/JsonProcessorLocalityMixIn.java => storage/blobstore/index/serde/JsonFileMetadataMixin.java} (53%)
 copy samza-core/src/main/java/org/apache/samza/{serializers/model/JsonProcessorLocalityMixIn.java => storage/blobstore/index/serde/JsonSnapshotIndexMixin.java} (51%)
 copy samza-core/src/main/java/org/apache/samza/{serializers/model/JsonTaskModelMixIn.java => storage/blobstore/index/serde/JsonSnapshotMetadataMixin.java} (57%)
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/index/serde/SnapshotIndexSerde.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/metrics/BlobStoreBackupManagerMetrics.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/metrics/BlobStoreRestoreManagerMetrics.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/util/BlobStoreUtil.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/storage/blobstore/util/DirDiffUtil.java
 create mode 100644 samza-core/src/main/java/org/apache/samza/util/FutureUtil.java
 rename samza-core/src/main/scala/org/apache/samza/serializers/{CheckpointSerde.scala => CheckpointV1Serde.scala} (54%)
 create mode 100644 samza-core/src/main/scala/org/apache/samza/storage/KafkaNonTransactionalStateTaskBackupManager.scala
 create mode 100644 samza-core/src/main/scala/org/apache/samza/storage/KafkaTransactionalStateTaskBackupManager.scala
 delete mode 100644 samza-core/src/main/scala/org/apache/samza/storage/NonTransactionalStateTaskStorageManager.scala
 delete mode 100644 samza-core/src/main/scala/org/apache/samza/storage/TaskStorageManager.scala
 delete mode 100644 samza-core/src/main/scala/org/apache/samza/storage/TaskStorageManagerFactory.java
 delete mode 100644 samza-core/src/main/scala/org/apache/samza/storage/TransactionalStateTaskStorageManager.scala
 create mode 100644 samza-core/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaChangelogSSPOffset.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/checkpoint/kafka/TestKafkaStateCheckpointMarker.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/serializers/TestCheckpointV2Serde.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/storage/TestKafkaChangelogStateBackendFactory.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/storage/TestTaskStorageCommitManager.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/TestBlobStoreBackupManager.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/TestBlobStoreRestoreManager.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/serde/TestSnapshotIndexSerde.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/util/BlobStoreTestUtil.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestBlobStoreUtil.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestDirDiffUtil.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/storage/blobstore/util/TestDirDiffUtilMisc.java
 create mode 100644 samza-core/src/test/java/org/apache/samza/util/TestFutureUtil.java
 rename samza-core/src/test/scala/org/apache/samza/serializers/{TestCheckpointSerde.scala => TestCheckpointV1Serde.scala} (69%)
 delete mode 100644 samza-core/src/test/scala/org/apache/samza/storage/TestTransactionalStateTaskStorageManager.java
 rename {samza-core/src/test/scala => samza-kafka/src/test/java}/org/apache/samza/storage/TestTaskStorageManager.scala (83%)
 create mode 100644 samza-kafka/src/test/java/org/apache/samza/storage/TestTransactionalStateTaskBackupManager.java
 create mode 100644 samza-test/src/test/java/org/apache/samza/checkpoint/CheckpointVersionIntegrationTest.java
 create mode 100644 samza-test/src/test/java/org/apache/samza/storage/MyStatefulApplication.java

[samza] 01/01: Merge pull request #1514 from apache/state-backend-async-commit

Posted by pm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

pmaheshwari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git

commit f40402948f37d84c000f1b4340180e538ac75c82
Merge: bad9ee8 c95d828
Author: Prateek Maheshwari <pr...@utexas.edu>
AuthorDate: Tue Aug 10 10:57:32 2021 -0700

    Merge pull request #1514 from apache/state-backend-async-commit
    
    All changes for
    SEP-28: https://cwiki.apache.org/confluence/display/SAMZA/SEP-28%3A+Samza+State+Backend+Interface+and+Checkpointing+Improvements
    SEP-29: https://cwiki.apache.org/confluence/display/SAMZA/SEP-29%3A+Blob+Store+Based+State+Backup+And+Restore

 build.gradle                                       |   1 +
 gradle/dependency-versions.gradle                  |   2 +-
 .../org/apache/samza/checkpoint/Checkpoint.java    |  57 +-
 .../org/apache/samza/checkpoint/CheckpointId.java  |  51 +-
 .../{Checkpoint.java => CheckpointV1.java}         |  37 +-
 .../org/apache/samza/checkpoint/CheckpointV2.java  | 123 +++
 .../java/org/apache/samza/job/model/JobModel.java  |   4 +
 .../samza/storage/BlobStoreAdminFactory.java       |  30 +-
 .../samza/storage/KafkaChangelogRestoreParams.java |  77 ++
 .../apache/samza/storage/StateBackendAdmin.java    |  32 +-
 .../apache/samza/storage/StateBackendFactory.java  |  63 ++
 .../org/apache/samza/storage/StorageEngine.java    |   8 +
 .../org/apache/samza/storage/StoreProperties.java  |  23 +-
 .../apache/samza/storage/TaskBackupManager.java    |  92 ++
 .../apache/samza/storage/TaskRestoreManager.java   |  14 +-
 .../samza/storage/blobstore/BlobStoreManager.java  |  83 ++
 .../storage/blobstore/BlobStoreManagerFactory.java |  24 +-
 .../apache/samza/storage/blobstore/Metadata.java   | 114 +++
 .../blobstore/exceptions/DeletedException.java     |  33 +-
 .../blobstore/exceptions/RetriableException.java   |  25 +-
 .../org/apache/samza/storage/kv/KeyValueStore.java |  13 +
 .../apache/samza/system/ChangelogSSPIterator.java  |   2 +-
 .../org/apache/samza/system/SystemFactory.java     |   2 +-
 .../apache/samza/checkpoint/TestCheckpointId.java  |  53 ++
 .../checkpoint/azure/AzureCheckpointManager.java   |  10 +-
 .../azure/ITestAzureCheckpointManager.java         |  15 +-
 .../checkpoint/kafka/KafkaChangelogSSPOffset.java  |  40 +-
 .../kafka/KafkaStateCheckpointMarker.java          | 145 ++++
 .../clustermanager/ClusterBasedJobCoordinator.java |  15 +
 .../org/apache/samza/config/BlobStoreConfig.java}  |  30 +-
 .../java/org/apache/samza/config/JobConfig.java    |  28 +
 .../org/apache/samza/config/StorageConfig.java     | 124 ++-
 .../java/org/apache/samza/config/TaskConfig.java   |  45 +
 .../samza/coordinator/MetadataResourceUtil.java    |   2 +-
 .../samza/serializers/CheckpointV2Serde.java       |  57 ++
 .../apache/samza/serializers/JsonCheckpoint.java   |  57 ++
 .../serializers/model/JsonCheckpointV2Mixin.java   |  46 +
 .../model/KafkaStateCheckpointMarkerMixin.java     |  48 ++
 .../samza/serializers/model/SamzaObjectMapper.java |  23 +
 .../standalone/PassthroughJobCoordinator.java      |  14 +-
 .../apache/samza/startpoint/StartpointManager.java |   2 +-
 .../storage/KafkaChangelogStateBackendFactory.java | 236 ++++++
 .../NonTransactionalStateTaskRestoreManager.java   |  81 +-
 .../apache/samza/storage/StorageManagerUtil.java   |  63 +-
 .../org/apache/samza/storage/StorageRecovery.java  |  16 +-
 .../samza/storage/TaskRestoreManagerFactory.java   |  82 --
 .../samza/storage/TaskSideInputStorageManager.java |  18 +-
 .../samza/storage/TaskStorageCommitManager.java    | 366 ++++++++
 .../TransactionalStateTaskRestoreManager.java      | 130 ++-
 .../storage/blobstore/BlobStoreBackupManager.java  | 350 ++++++++
 .../storage/blobstore/BlobStoreRestoreManager.java | 367 ++++++++
 .../blobstore/BlobStoreStateBackendFactory.java    | 100 +++
 .../samza/storage/blobstore/diff/DirDiff.java      | 244 ++++++
 .../samza/storage/blobstore/index/DirIndex.java    | 176 ++++
 .../samza/storage/blobstore/index/FileBlob.java    |  85 ++
 .../samza/storage/blobstore/index/FileIndex.java   | 114 +++
 .../storage/blobstore/index/FileMetadata.java      | 138 +++
 .../storage/blobstore/index/SnapshotIndex.java     | 124 +++
 .../storage/blobstore/index/SnapshotMetadata.java  | 115 +++
 .../blobstore/index/serde/JsonDirIndexMixin.java   |  58 ++
 .../blobstore/index/serde/JsonFileBlobMixin.java}  |  34 +-
 .../blobstore/index/serde/JsonFileIndexMixin.java  |  53 ++
 .../index/serde/JsonFileMetadataMixin.java         |  55 ++
 .../index/serde/JsonSnapshotIndexMixin.java        |  53 ++
 .../index/serde/JsonSnapshotMetadataMixin.java     |  53 ++
 .../blobstore/index/serde/SnapshotIndexSerde.java  |  74 ++
 .../metrics/BlobStoreBackupManagerMetrics.java     | 136 +++
 .../metrics/BlobStoreRestoreManagerMetrics.java    |  94 +++
 .../storage/blobstore/util/BlobStoreUtil.java      | 622 ++++++++++++++
 .../samza/storage/blobstore/util/DirDiffUtil.java  | 403 +++++++++
 .../java/org/apache/samza/system/SystemAdmins.java |   8 +
 .../samza/system/inmemory/InMemoryManager.java     |   2 +-
 .../samza/table/retry/AsyncRetriableTable.java     |  11 +-
 .../apache/samza/table/retry/FailsafeAdapter.java  |  43 +-
 .../java/org/apache/samza/util/FutureUtil.java     | 191 +++++
 .../apache/samza/checkpoint/CheckpointTool.scala   |  29 +-
 .../apache/samza/checkpoint/OffsetManager.scala    |  30 +-
 .../file/FileSystemCheckpointManager.scala         |  13 +-
 .../apache/samza/container/SamzaContainer.scala    | 161 ++--
 .../org/apache/samza/container/TaskInstance.scala  | 324 +++++--
 .../samza/container/TaskInstanceMetrics.scala      |  12 +-
 .../apache/samza/job/local/ProcessJobFactory.scala |  15 +-
 ...eckpointSerde.scala => CheckpointV1Serde.scala} |  69 +-
 .../samza/storage/ContainerStorageManager.java     | 448 ++++++----
 ...fkaNonTransactionalStateTaskBackupManager.scala |  91 ++
 .../KafkaTransactionalStateTaskBackupManager.scala |  97 +++
 .../NonTransactionalStateTaskStorageManager.scala  | 144 ----
 .../samza/storage/TaskStorageManagerFactory.java   |  46 -
 .../TransactionalStateTaskStorageManager.scala     | 201 -----
 .../scala/org/apache/samza/util/FileUtil.scala     |   9 +-
 .../kafka/TestKafkaChangelogSSPOffset.java         |  73 ++
 .../kafka/TestKafkaStateCheckpointMarker.java      | 114 +++
 .../org/apache/samza/config/TestStorageConfig.java | 259 +++++-
 .../org/apache/samza/job/model/TestJobModel.java   |   2 +-
 .../samza/serializers/TestCheckpointV2Serde.java   |  96 +++
 .../TestKafkaChangelogStateBackendFactory.java     |  80 ++
 .../storage/TestTaskStorageCommitManager.java      | 916 ++++++++++++++++++++
 .../TestTransactionalStateTaskRestoreManager.java  | 252 +++---
 .../blobstore/TestBlobStoreBackupManager.java      | 542 ++++++++++++
 .../blobstore/TestBlobStoreRestoreManager.java     | 362 ++++++++
 .../blobstore/serde/TestSnapshotIndexSerde.java    |  56 ++
 .../storage/blobstore/util/BlobStoreTestUtil.java  | 296 +++++++
 .../storage/blobstore/util/TestBlobStoreUtil.java  | 936 +++++++++++++++++++++
 .../storage/blobstore/util/TestDirDiffUtil.java    | 127 +++
 .../blobstore/util/TestDirDiffUtilMisc.java        |  71 ++
 .../java/org/apache/samza/util/TestFutureUtil.java | 222 +++++
 .../samza/checkpoint/TestCheckpointTool.scala      |  66 +-
 .../samza/checkpoint/TestOffsetManager.scala       |  30 +-
 .../file/TestFileSystemCheckpointManager.scala     |  10 +-
 .../samza/container/TestSamzaContainer.scala       |  34 -
 .../apache/samza/container/TestTaskInstance.scala  | 742 ++++++++++++++--
 ...ointSerde.scala => TestCheckpointV1Serde.scala} |  26 +-
 .../samza/storage/TestContainerStorageManager.java | 266 +++++-
 .../TestTransactionalStateTaskStorageManager.java  | 534 ------------
 .../scala/org/apache/samza/util/TestFileUtil.scala |  49 +-
 .../checkpoint/kafka/KafkaCheckpointLogKey.java    |   9 +-
 .../kafka/KafkaCheckpointLogKeySerde.java          |   7 +-
 .../samza/system/kafka/KafkaConsumerProxy.java     |   5 +-
 .../checkpoint/kafka/KafkaCheckpointManager.scala  | 123 ++-
 .../kafka/TestKafkaCheckpointLogKeySerde.java      |  14 +-
 .../kafka/TestKafkaCheckpointManagerJava.java      |  29 +-
 .../samza/storage/TestTaskStorageManager.scala     | 215 ++---
 .../TestTransactionalStateTaskBackupManager.java   | 276 ++++++
 .../kafka/TestKafkaCheckpointManager.scala         | 230 ++++-
 .../InMemoryKeyValueStorageEngineFactory.java      |   2 -
 .../samza/storage/kv/RocksDbOptionsHelper.java     | 124 ++-
 .../kv/RocksDbKeyValueStorageEngineFactory.scala   |  10 +-
 .../samza/storage/kv/RocksDbKeyValueStore.scala    |  19 +-
 .../storage/kv/TestRocksDbKeyValueStore.scala      |  20 +-
 .../kv/BaseKeyValueStorageEngineFactory.java       |   9 +-
 .../kv/MockKeyValueStorageEngineFactory.java       |   4 +-
 .../kv/TestBaseKeyValueStorageEngineFactory.java   |  37 +-
 .../CheckpointVersionIntegrationTest.java          | 149 ++++
 .../samza/storage/MyStatefulApplication.java       | 157 ++++
 .../kv/TransactionalStateIntegrationTest.java      | 135 +--
 ...ransactionalStateMultiStoreIntegrationTest.java | 120 +--
 .../processor/TestZkLocalApplicationRunner.java    |   4 +-
 .../NonTransactionalStateIntegrationTest.scala     |   5 +-
 .../test/integration/StreamTaskTestUtil.scala      |   8 +-
 139 files changed, 13422 insertions(+), 2498 deletions(-)