You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by pm...@apache.org on 2021/08/10 17:57:40 UTC
[samza] 01/01: Merge pull request #1514 from
apache/state-backend-async-commit
This is an automated email from the ASF dual-hosted git repository.
pmaheshwari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git
commit f40402948f37d84c000f1b4340180e538ac75c82
Merge: bad9ee8 c95d828
Author: Prateek Maheshwari <pr...@utexas.edu>
AuthorDate: Tue Aug 10 10:57:32 2021 -0700
Merge pull request #1514 from apache/state-backend-async-commit
All changes for
SEP-28: https://cwiki.apache.org/confluence/display/SAMZA/SEP-28%3A+Samza+State+Backend+Interface+and+Checkpointing+Improvements
SEP-29: https://cwiki.apache.org/confluence/display/SAMZA/SEP-29%3A+Blob+Store+Based+State+Backup+And+Restore
build.gradle | 1 +
gradle/dependency-versions.gradle | 2 +-
.../org/apache/samza/checkpoint/Checkpoint.java | 57 +-
.../org/apache/samza/checkpoint/CheckpointId.java | 51 +-
.../{Checkpoint.java => CheckpointV1.java} | 37 +-
.../org/apache/samza/checkpoint/CheckpointV2.java | 123 +++
.../java/org/apache/samza/job/model/JobModel.java | 4 +
.../samza/storage/BlobStoreAdminFactory.java | 30 +-
.../samza/storage/KafkaChangelogRestoreParams.java | 77 ++
.../apache/samza/storage/StateBackendAdmin.java | 32 +-
.../apache/samza/storage/StateBackendFactory.java | 63 ++
.../org/apache/samza/storage/StorageEngine.java | 8 +
.../org/apache/samza/storage/StoreProperties.java | 23 +-
.../apache/samza/storage/TaskBackupManager.java | 92 ++
.../apache/samza/storage/TaskRestoreManager.java | 14 +-
.../samza/storage/blobstore/BlobStoreManager.java | 83 ++
.../storage/blobstore/BlobStoreManagerFactory.java | 24 +-
.../apache/samza/storage/blobstore/Metadata.java | 114 +++
.../blobstore/exceptions/DeletedException.java | 33 +-
.../blobstore/exceptions/RetriableException.java | 25 +-
.../org/apache/samza/storage/kv/KeyValueStore.java | 13 +
.../apache/samza/system/ChangelogSSPIterator.java | 2 +-
.../org/apache/samza/system/SystemFactory.java | 2 +-
.../apache/samza/checkpoint/TestCheckpointId.java | 53 ++
.../checkpoint/azure/AzureCheckpointManager.java | 10 +-
.../azure/ITestAzureCheckpointManager.java | 15 +-
.../checkpoint/kafka/KafkaChangelogSSPOffset.java | 40 +-
.../kafka/KafkaStateCheckpointMarker.java | 145 ++++
.../clustermanager/ClusterBasedJobCoordinator.java | 15 +
.../org/apache/samza/config/BlobStoreConfig.java} | 30 +-
.../java/org/apache/samza/config/JobConfig.java | 28 +
.../org/apache/samza/config/StorageConfig.java | 124 ++-
.../java/org/apache/samza/config/TaskConfig.java | 45 +
.../samza/coordinator/MetadataResourceUtil.java | 2 +-
.../samza/serializers/CheckpointV2Serde.java | 57 ++
.../apache/samza/serializers/JsonCheckpoint.java | 57 ++
.../serializers/model/JsonCheckpointV2Mixin.java | 46 +
.../model/KafkaStateCheckpointMarkerMixin.java | 48 ++
.../samza/serializers/model/SamzaObjectMapper.java | 23 +
.../standalone/PassthroughJobCoordinator.java | 14 +-
.../apache/samza/startpoint/StartpointManager.java | 2 +-
.../storage/KafkaChangelogStateBackendFactory.java | 236 ++++++
.../NonTransactionalStateTaskRestoreManager.java | 81 +-
.../apache/samza/storage/StorageManagerUtil.java | 63 +-
.../org/apache/samza/storage/StorageRecovery.java | 16 +-
.../samza/storage/TaskRestoreManagerFactory.java | 82 --
.../samza/storage/TaskSideInputStorageManager.java | 18 +-
.../samza/storage/TaskStorageCommitManager.java | 366 ++++++++
.../TransactionalStateTaskRestoreManager.java | 130 ++-
.../storage/blobstore/BlobStoreBackupManager.java | 350 ++++++++
.../storage/blobstore/BlobStoreRestoreManager.java | 367 ++++++++
.../blobstore/BlobStoreStateBackendFactory.java | 100 +++
.../samza/storage/blobstore/diff/DirDiff.java | 244 ++++++
.../samza/storage/blobstore/index/DirIndex.java | 176 ++++
.../samza/storage/blobstore/index/FileBlob.java | 85 ++
.../samza/storage/blobstore/index/FileIndex.java | 114 +++
.../storage/blobstore/index/FileMetadata.java | 138 +++
.../storage/blobstore/index/SnapshotIndex.java | 124 +++
.../storage/blobstore/index/SnapshotMetadata.java | 115 +++
.../blobstore/index/serde/JsonDirIndexMixin.java | 58 ++
.../blobstore/index/serde/JsonFileBlobMixin.java} | 34 +-
.../blobstore/index/serde/JsonFileIndexMixin.java | 53 ++
.../index/serde/JsonFileMetadataMixin.java | 55 ++
.../index/serde/JsonSnapshotIndexMixin.java | 53 ++
.../index/serde/JsonSnapshotMetadataMixin.java | 53 ++
.../blobstore/index/serde/SnapshotIndexSerde.java | 74 ++
.../metrics/BlobStoreBackupManagerMetrics.java | 136 +++
.../metrics/BlobStoreRestoreManagerMetrics.java | 94 +++
.../storage/blobstore/util/BlobStoreUtil.java | 622 ++++++++++++++
.../samza/storage/blobstore/util/DirDiffUtil.java | 403 +++++++++
.../java/org/apache/samza/system/SystemAdmins.java | 8 +
.../samza/system/inmemory/InMemoryManager.java | 2 +-
.../samza/table/retry/AsyncRetriableTable.java | 11 +-
.../apache/samza/table/retry/FailsafeAdapter.java | 43 +-
.../java/org/apache/samza/util/FutureUtil.java | 191 +++++
.../apache/samza/checkpoint/CheckpointTool.scala | 29 +-
.../apache/samza/checkpoint/OffsetManager.scala | 30 +-
.../file/FileSystemCheckpointManager.scala | 13 +-
.../apache/samza/container/SamzaContainer.scala | 161 ++--
.../org/apache/samza/container/TaskInstance.scala | 324 +++++--
.../samza/container/TaskInstanceMetrics.scala | 12 +-
.../apache/samza/job/local/ProcessJobFactory.scala | 15 +-
...eckpointSerde.scala => CheckpointV1Serde.scala} | 69 +-
.../samza/storage/ContainerStorageManager.java | 448 ++++++----
...fkaNonTransactionalStateTaskBackupManager.scala | 91 ++
.../KafkaTransactionalStateTaskBackupManager.scala | 97 +++
.../NonTransactionalStateTaskStorageManager.scala | 144 ----
.../samza/storage/TaskStorageManagerFactory.java | 46 -
.../TransactionalStateTaskStorageManager.scala | 201 -----
.../scala/org/apache/samza/util/FileUtil.scala | 9 +-
.../kafka/TestKafkaChangelogSSPOffset.java | 73 ++
.../kafka/TestKafkaStateCheckpointMarker.java | 114 +++
.../org/apache/samza/config/TestStorageConfig.java | 259 +++++-
.../org/apache/samza/job/model/TestJobModel.java | 2 +-
.../samza/serializers/TestCheckpointV2Serde.java | 96 +++
.../TestKafkaChangelogStateBackendFactory.java | 80 ++
.../storage/TestTaskStorageCommitManager.java | 916 ++++++++++++++++++++
.../TestTransactionalStateTaskRestoreManager.java | 252 +++---
.../blobstore/TestBlobStoreBackupManager.java | 542 ++++++++++++
.../blobstore/TestBlobStoreRestoreManager.java | 362 ++++++++
.../blobstore/serde/TestSnapshotIndexSerde.java | 56 ++
.../storage/blobstore/util/BlobStoreTestUtil.java | 296 +++++++
.../storage/blobstore/util/TestBlobStoreUtil.java | 936 +++++++++++++++++++++
.../storage/blobstore/util/TestDirDiffUtil.java | 127 +++
.../blobstore/util/TestDirDiffUtilMisc.java | 71 ++
.../java/org/apache/samza/util/TestFutureUtil.java | 222 +++++
.../samza/checkpoint/TestCheckpointTool.scala | 66 +-
.../samza/checkpoint/TestOffsetManager.scala | 30 +-
.../file/TestFileSystemCheckpointManager.scala | 10 +-
.../samza/container/TestSamzaContainer.scala | 34 -
.../apache/samza/container/TestTaskInstance.scala | 742 ++++++++++++++--
...ointSerde.scala => TestCheckpointV1Serde.scala} | 26 +-
.../samza/storage/TestContainerStorageManager.java | 266 +++++-
.../TestTransactionalStateTaskStorageManager.java | 534 ------------
.../scala/org/apache/samza/util/TestFileUtil.scala | 49 +-
.../checkpoint/kafka/KafkaCheckpointLogKey.java | 9 +-
.../kafka/KafkaCheckpointLogKeySerde.java | 7 +-
.../samza/system/kafka/KafkaConsumerProxy.java | 5 +-
.../checkpoint/kafka/KafkaCheckpointManager.scala | 123 ++-
.../kafka/TestKafkaCheckpointLogKeySerde.java | 14 +-
.../kafka/TestKafkaCheckpointManagerJava.java | 29 +-
.../samza/storage/TestTaskStorageManager.scala | 215 ++---
.../TestTransactionalStateTaskBackupManager.java | 276 ++++++
.../kafka/TestKafkaCheckpointManager.scala | 230 ++++-
.../InMemoryKeyValueStorageEngineFactory.java | 2 -
.../samza/storage/kv/RocksDbOptionsHelper.java | 124 ++-
.../kv/RocksDbKeyValueStorageEngineFactory.scala | 10 +-
.../samza/storage/kv/RocksDbKeyValueStore.scala | 19 +-
.../storage/kv/TestRocksDbKeyValueStore.scala | 20 +-
.../kv/BaseKeyValueStorageEngineFactory.java | 9 +-
.../kv/MockKeyValueStorageEngineFactory.java | 4 +-
.../kv/TestBaseKeyValueStorageEngineFactory.java | 37 +-
.../CheckpointVersionIntegrationTest.java | 149 ++++
.../samza/storage/MyStatefulApplication.java | 157 ++++
.../kv/TransactionalStateIntegrationTest.java | 135 +--
...ransactionalStateMultiStoreIntegrationTest.java | 120 +--
.../processor/TestZkLocalApplicationRunner.java | 4 +-
.../NonTransactionalStateIntegrationTest.scala | 5 +-
.../test/integration/StreamTaskTestUtil.scala | 8 +-
139 files changed, 13422 insertions(+), 2498 deletions(-)