You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2025/01/17 07:00:00 UTC
(hadoop) 01/01: HADOOP-19236. Incorporate VolcanoEngine Cloud TOS File System Implementation.
This is an automated email from the ASF dual-hosted git repository.
jinglun pushed a commit to branch HADOOP-19236-01
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 0d3b07ceee5a2dd869099a43ef80b503323c59fe
Author: lijinglun <li...@bytedance.com>
AuthorDate: Mon Aug 12 19:36:28 2024 +0800
HADOOP-19236. Incorporate VolcanoEngine Cloud TOS File System Implementation.
Contributed by: ZhengHu, SunXin, XianyinXin, Rascal Wu, FangBo, Yuanzhihuan.
---
.../hadoop-cloud-storage/pom.xml | 11 +
.../hadoop-tos/hadoop-tos-core/pom.xml | 167 +++
.../java/org/apache/hadoop/fs/tosfs/RawFS.java | 47 +
.../org/apache/hadoop/fs/tosfs/RawFSUtils.java | 59 +
.../org/apache/hadoop/fs/tosfs/RawFileStatus.java | 50 +
.../org/apache/hadoop/fs/tosfs/RawFileSystem.java | 751 ++++++++++++
.../hadoop/fs/tosfs/RawLocatedFileStatus.java | 30 +
.../org/apache/hadoop/fs/tosfs/TosChecksum.java | 67 ++
.../java/org/apache/hadoop/fs/tosfs/TosFS.java | 47 +
.../org/apache/hadoop/fs/tosfs/TosFileSystem.java | 43 +
.../hadoop/fs/tosfs/commit/CommitContext.java | 45 +
.../apache/hadoop/fs/tosfs/commit/CommitUtils.java | 359 ++++++
.../apache/hadoop/fs/tosfs/commit/Committer.java | 499 ++++++++
.../hadoop/fs/tosfs/commit/CommitterFactory.java | 33 +
.../hadoop/fs/tosfs/commit/MagicOutputStream.java | 146 +++
.../org/apache/hadoop/fs/tosfs/commit/Pending.java | 181 +++
.../apache/hadoop/fs/tosfs/commit/PendingSet.java | 123 ++
.../apache/hadoop/fs/tosfs/commit/SuccessData.java | 238 ++++
.../hadoop/fs/tosfs/commit/mapred/Committer.java | 184 +++
.../hadoop/fs/tosfs/commit/ops/PendingOps.java | 43 +
.../fs/tosfs/commit/ops/PendingOpsFactory.java | 40 +
.../hadoop/fs/tosfs/commit/ops/RawPendingOps.java | 54 +
.../org/apache/hadoop/fs/tosfs/common/Bytes.java | 182 +++
.../org/apache/hadoop/fs/tosfs/common/Chain.java | 147 +++
.../org/apache/hadoop/fs/tosfs/common/Tasks.java | 590 ++++++++++
.../apache/hadoop/fs/tosfs/common/ThreadPools.java | 142 +++
.../apache/hadoop/fs/tosfs/conf/ArgumentKey.java | 32 +
.../org/apache/hadoop/fs/tosfs/conf/ConfKeys.java | 154 +++
.../apache/hadoop/fs/tosfs/conf/FileStoreKeys.java | 39 +
.../org/apache/hadoop/fs/tosfs/conf/TosKeys.java | 274 +++++
.../apache/hadoop/fs/tosfs/object/BucketInfo.java | 91 ++
.../hadoop/fs/tosfs/object/ChecksumInfo.java | 37 +
.../hadoop/fs/tosfs/object/ChecksumType.java | 61 +
.../apache/hadoop/fs/tosfs/object/Constants.java | 32 +
.../hadoop/fs/tosfs/object/DirectoryStorage.java | 50 +
.../apache/hadoop/fs/tosfs/object/FileStore.java | 632 ++++++++++
.../fs/tosfs/object/InputStreamProvider.java | 34 +
.../hadoop/fs/tosfs/object/MultipartUpload.java | 102 ++
.../hadoop/fs/tosfs/object/ObjectConstants.java | 27 +
.../hadoop/fs/tosfs/object/ObjectContent.java | 52 +
.../apache/hadoop/fs/tosfs/object/ObjectInfo.java | 117 ++
.../tosfs/object/ObjectMultiRangeInputStream.java | 233 ++++
.../hadoop/fs/tosfs/object/ObjectOutputStream.java | 336 ++++++
.../fs/tosfs/object/ObjectRangeInputStream.java | 198 ++++
.../hadoop/fs/tosfs/object/ObjectStorage.java | 367 ++++++
.../fs/tosfs/object/ObjectStorageFactory.java | 71 ++
.../apache/hadoop/fs/tosfs/object/ObjectUtils.java | 90 ++
.../org/apache/hadoop/fs/tosfs/object/Part.java | 78 ++
.../hadoop/fs/tosfs/object/PrefixStorage.java | 265 +++++
.../exceptions/ChecksumMismatchException.java | 16 +
.../exceptions/InvalidObjectKeyException.java | 31 +
.../object/exceptions/NotAppendableException.java | 25 +
.../tosfs/object/request/ListObjectsRequest.java | 85 ++
.../tosfs/object/response/ListObjectsResponse.java | 43 +
.../fs/tosfs/object/staging/FileStagingPart.java | 176 +++
.../fs/tosfs/object/staging/StagingPart.java | 79 ++
.../hadoop/fs/tosfs/object/staging/State.java | 23 +
.../fs/tosfs/object/tos/ChainTOSInputStream.java | 136 +++
.../fs/tosfs/object/tos/DelegationClient.java | 1230 ++++++++++++++++++++
.../tosfs/object/tos/DelegationClientBuilder.java | 183 +++
.../fs/tosfs/object/tos/GetObjectOutput.java | 60 +
.../org/apache/hadoop/fs/tosfs/object/tos/TOS.java | 1038 +++++++++++++++++
.../hadoop/fs/tosfs/object/tos/TOSErrorCodes.java | 56 +
.../hadoop/fs/tosfs/object/tos/TOSInputStream.java | 118 ++
.../hadoop/fs/tosfs/object/tos/TOSUtils.java | 116 ++
.../hadoop/fs/tosfs/object/tos/TosObjectInfo.java | 78 ++
.../tos/auth/AbstractCredentialsProvider.java | 76 ++
.../tosfs/object/tos/auth/CredentialsProvider.java | 35 +
.../tos/auth/DefaultCredentialsProviderChain.java | 100 ++
.../tos/auth/EnvironmentCredentialsProvider.java | 34 +
.../object/tos/auth/ExpireableCredential.java | 67 ++
.../object/tos/auth/SimpleCredentialsProvider.java | 58 +
.../apache/hadoop/fs/tosfs/ops/DefaultFsOps.java | 187 +++
.../apache/hadoop/fs/tosfs/ops/DirectoryFsOps.java | 108 ++
.../java/org/apache/hadoop/fs/tosfs/ops/FsOps.java | 88 ++
.../org/apache/hadoop/fs/tosfs/ops/RenameOp.java | 223 ++++
.../apache/hadoop/fs/tosfs/util/CommonUtils.java | 44 +
.../org/apache/hadoop/fs/tosfs/util/FSUtils.java | 76 ++
.../org/apache/hadoop/fs/tosfs/util/FuseUtils.java | 28 +
.../org/apache/hadoop/fs/tosfs/util/Iterables.java | 126 ++
.../org/apache/hadoop/fs/tosfs/util/JsonCodec.java | 46 +
.../apache/hadoop/fs/tosfs/util/LazyReload.java | 35 +
.../hadoop/fs/tosfs/util/LazyReloadIter.java | 69 ++
.../apache/hadoop/fs/tosfs/util/ParseUtils.java | 181 +++
.../org/apache/hadoop/fs/tosfs/util/Range.java | 121 ++
.../org/apache/hadoop/fs/tosfs/util/Reload.java | 29 +
.../hadoop/fs/tosfs/util/RemoteIterators.java | 122 ++
.../hadoop/fs/tosfs/util/RetryableUtils.java | 54 +
.../apache/hadoop/fs/tosfs/util/Serializer.java | 25 +
.../fs/tosfs/util/TOSClientContextUtils.java | 59 +
.../org/apache/hadoop/fs/tosfs/util/UUIDUtils.java | 31 +
.../src/site/markdown/cloud-storage/index.md | 425 +++++++
.../hadoop-tos/hadoop-tos-shade/pom.xml | 132 +++
.../{ => hadoop-tos}/pom.xml | 46 +-
hadoop-cloud-storage-project/pom.xml | 1 +
hadoop-project/pom.xml | 18 +
hadoop-project/src/site/site.xml | 1 +
pom.xml | 7 +
98 files changed, 13767 insertions(+), 28 deletions(-)
diff --git a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
index 23206c0a70f..bb080fcdd82 100644
--- a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
@@ -136,6 +136,17 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-huaweicloud</artifactId>
<scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.squareup.okhttp3</groupId>
+ <artifactId>okhttp</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-tos</artifactId>
+ <scope>compile</scope>
</dependency>
</dependencies>
</project>
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/pom.xml b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/pom.xml
new file mode 100644
index 00000000000..c6f04f30bec
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/pom.xml
@@ -0,0 +1,167 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>3.5.0-SNAPSHOT</version>
+ <relativePath>../../../hadoop-project/pom.xml</relativePath>
+ </parent>
+
+ <artifactId>hadoop-tos-core</artifactId>
+ <version>3.5.0-SNAPSHOT</version>
+ <name>Apache Hadoop Volcano Engine Services Core</name>
+ <description>
+ This module contains code to support integration with Volcano Engine TOS.
+ It also declares the dependencies needed to work with Volcano Engine services.
+ </description>
+ <packaging>jar</packaging>
+
+ <properties>
+ <file.encoding>UTF-8</file.encoding>
+ <ve-tos-java-sdk.version>2.8.6</ve-tos-java-sdk.version>
+ <tos.shading.prefix>org.apache.hadoop.fs.tosfs.shaded</tos.shading.prefix>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <!-- The ve-tos-java-sdk 2.8.6 depends on okhttp, but okhttp violates dependency convergence.
+ The com.squareup.okhttp3:okhttp:jar:4.10.0 depends on org.jetbrains.kotlin:kotlin-stdlib-common,
+ with 2 different versions: 1.5.31 and 1.6.20. Force configure it to 1.6.20. -->
+ <dependency>
+ <groupId>org.jetbrains.kotlin</groupId>
+ <artifactId>kotlin-stdlib-common</artifactId>
+ <version>1.6.20</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-tos-shade</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.volcengine</groupId>
+ <artifactId>ve-tos-java-sdk</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <!-- Artifacts needed to bring up a Mini MR Yarn cluster-->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-examples</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>4.11.0</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.21.0</version>
+ <configuration>
+ <parallel>classes</parallel>
+ <threadCount>1</threadCount>
+ <perCoreThreadCount>true</perCoreThreadCount>
+ <rerunFailingTestsCount>2</rerunFailingTestsCount>
+ <forkCount>8</forkCount>
+ <argLine>-Xmx2048m</argLine>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${maven-shade-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>package-dependency</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeSourcesContent>true</shadeSourcesContent>
+ <createSourcesJar>true</createSourcesJar>
+ <artifactSet>
+ <includes>
+ <include>org.apache.hadoop:hadoop-tos-shade</include>
+ </includes>
+ </artifactSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawFS.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawFS.java
new file mode 100644
index 00000000000..6fba99b3d61
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawFS.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * The implementation class of the raw AbstractFileSystem. If you want to use object storage as
+ * YARN’s resource storage dir via the fs.defaultFS configuration property in Hadoop’s
+ * core-site.xml, you should add this configuration to Hadoop's core-site.xml.
+ * <pre>
+ * fs.AbstractFileSystem.{scheme}.impl=io.proton.fs.RawFS.
+ * </pre>
+ */
+public class RawFS extends DelegateToFileSystem {
+ private static final int TOS_DEFAULT_PORT = -1;
+
+ public RawFS(URI uri, Configuration conf) throws IOException, URISyntaxException {
+ super(uri, new RawFileSystem(), conf, uri.getScheme(), false);
+ }
+
+ @Override
+ public int getUriDefaultPort() {
+ return TOS_DEFAULT_PORT;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawFSUtils.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawFSUtils.java
new file mode 100644
index 00000000000..e8cd9c7b793
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawFSUtils.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Preconditions;
+
+import java.util.Objects;
+
+public class RawFSUtils {
+ private RawFSUtils() {
+ }
+
+ /**
+ * @return true means the node is included in the subtree which has the root node.
+ */
+ public static boolean inSubtree(String root, String p) {
+ return inSubtree(new Path(root), new Path(p));
+ }
+
+ /**
+ * @return true means the node is included in the subtree which has the root node.
+ */
+ public static boolean inSubtree(Path root, Path node) {
+ Preconditions.checkNotNull(root, "Root cannot be null");
+ Preconditions.checkNotNull(node, "Node cannot be null");
+ if (root.isRoot()) {
+ return true;
+ }
+
+ if (Objects.equals(root, node)) {
+ return true;
+ }
+
+ while (!node.isRoot()) {
+ if (Objects.equals(root, node)) {
+ return true;
+ }
+ node = node.getParent();
+ }
+ return false;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawFileStatus.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawFileStatus.java
new file mode 100644
index 00000000000..99b40b78349
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawFileStatus.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.object.Constants;
+
+public class RawFileStatus extends FileStatus {
+ private final byte[] checksum;
+
+ /**
+ * File status of directory
+ *
+ * @param path directory path
+ * @param owner directory owner
+ */
+ public RawFileStatus(Path path, String owner) {
+ this(0, true, 1, System.currentTimeMillis(), path, owner, Constants.MAGIC_CHECKSUM);
+ }
+
+ public RawFileStatus(
+ long length, boolean isdir, long blocksize,
+ long modification_time, Path path, String owner, byte[] checksum) {
+ super(length, isdir, 1, blocksize, modification_time, path);
+ setOwner(owner);
+ setGroup(owner);
+ this.checksum = checksum;
+ }
+
+ public byte[] checksum() {
+ return checksum;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawFileSystem.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawFileSystem.java
new file mode 100644
index 00000000000..fe90bcba234
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawFileSystem.java
@@ -0,0 +1,751 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.tosfs.commit.MagicOutputStream;
+import org.apache.hadoop.fs.tosfs.common.Bytes;
+import org.apache.hadoop.fs.tosfs.common.ThreadPools;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.object.ChecksumInfo;
+import org.apache.hadoop.fs.tosfs.object.Constants;
+import org.apache.hadoop.fs.tosfs.object.DirectoryStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectInfo;
+import org.apache.hadoop.fs.tosfs.object.ObjectMultiRangeInputStream;
+import org.apache.hadoop.fs.tosfs.object.ObjectOutputStream;
+import org.apache.hadoop.fs.tosfs.object.ObjectRangeInputStream;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
+import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
+import org.apache.hadoop.fs.tosfs.object.exceptions.InvalidObjectKeyException;
+import org.apache.hadoop.fs.tosfs.ops.DefaultFsOps;
+import org.apache.hadoop.fs.tosfs.ops.DirectoryFsOps;
+import org.apache.hadoop.fs.tosfs.ops.FsOps;
+import org.apache.hadoop.fs.tosfs.util.FSUtils;
+import org.apache.hadoop.fs.tosfs.util.FuseUtils;
+import org.apache.hadoop.fs.tosfs.util.Range;
+import org.apache.hadoop.fs.tosfs.util.RemoteIterators;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Iterators;
+import org.apache.hadoop.util.DataChecksum;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
+import org.apache.hadoop.util.Progressable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Collections;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.fs.XAttrSetFlag.CREATE;
+import static org.apache.hadoop.fs.XAttrSetFlag.REPLACE;
+
+public class RawFileSystem extends FileSystem {
+ private static final Logger LOG = LoggerFactory.getLogger(RawFileSystem.class);
+ private static final String MULTIPART_THREAD_POOL_PREFIX = "rawfs-multipart-thread-pool";
+ private static final String TASK_THREAD_POOL_PREFIX = "rawfs-task-thread-pool";
+ // This is the same as HdfsClientConfigKeys.DFS_BLOCK_SIZE_KEY, we do not
+ // use that directly because we don't want to introduce the hdfs client library.
+ private static final String DFS_BLOCK_SIZE_KEY = "dfs.blocksize";
+ private static final long DFS_BLOCK_SIZE_DEFAULT = 128 << 20;
+
+ private String scheme;
+ private String username;
+ private Path workingDir;
+ private URI uri;
+ private String bucket;
+ private ObjectStorage storage;
+ // Use for task parallel execution, such as parallel to copy multiple files.
+ private ExecutorService taskThreadPool;
+ // Use for file multipart upload only.
+ private ExecutorService uploadThreadPool;
+ private FsOps fsOps;
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return scheme;
+ }
+
+ @VisibleForTesting
+ String bucket() {
+ return bucket;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ LOG.debug("Opening '{}' for reading.", path);
+ RawFileStatus status = innerFileStatus(path);
+ if (status.isDirectory()) {
+ throw new FileNotFoundException(String.format("Can't open %s because it is a directory", path));
+ }
+
+ // Parse the range size from the hadoop conf.
+ long rangeSize = getConf().getLong(
+ ConfKeys.FS_OBJECT_STREAM_RANGE_SIZE,
+ ConfKeys.FS_OBJECT_STREAM_RANGE_SIZE_DEFAULT);
+ Preconditions.checkArgument(rangeSize > 0, "Object storage range size must be positive.");
+
+ FSInputStream fsIn = new ObjectMultiRangeInputStream(taskThreadPool, storage, path,
+ status.getLen(), rangeSize, status.checksum());
+ return new FSDataInputStream(fsIn);
+ }
+
+ public FSDataInputStream open(Path path, byte[] expectedChecksum, Range range) throws IOException {
+ return new FSDataInputStream(new ObjectRangeInputStream(storage, path, range, expectedChecksum));
+ }
+
+ @Override
+ public FSDataOutputStream create(
+ Path path, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize,
+ Progressable progress) throws IOException {
+ FileStatus fileStatus = getFileStatusOrNull(path);
+ if (fileStatus != null) {
+ if (fileStatus.isDirectory()) {
+ throw new FileAlreadyExistsException(path + " is a directory");
+ }
+
+ if (!overwrite) {
+ throw new FileAlreadyExistsException(path + " already exists");
+ }
+ LOG.debug("Overwriting file {}", path);
+ }
+
+ if (MagicOutputStream.isMagic(path)) {
+ return new FSDataOutputStream(
+ new MagicOutputStream(this, storage, uploadThreadPool, getConf(), makeQualified(path)), null);
+ } else {
+ ObjectOutputStream out =
+ new ObjectOutputStream(storage, uploadThreadPool, getConf(), makeQualified(path), true);
+
+ if (fileStatus == null && FuseUtils.fuseEnabled()) {
+ // The fuse requires the file to be visible when accessing getFileStatus once we created the file, so here we
+ // close and commit the file to be visible explicitly for fuse, and then reopen the file output stream for
+ // further data bytes writing.
+ out.close();
+ out = new ObjectOutputStream(storage, uploadThreadPool, getConf(), makeQualified(path), true);
+ }
+
+ return new FSDataOutputStream(out, null);
+ }
+ }
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive) throws IOException {
+ Path path = makeQualified(f);
+ LOG.debug("listFiles({}, {})", path, recursive);
+
+ // assume the path is a dir at first, and list sub files
+ RemoteIterator<LocatedFileStatus> subFiles = RemoteIterators.fromIterable(
+ fsOps.listDir(path, recursive, key -> !ObjectInfo.isDir(key)), this::toLocatedFileStatus);
+ if (!subFiles.hasNext()) {
+ final RawFileStatus fileStatus = innerFileStatus(path);
+ if (fileStatus.isFile()) {
+ return RemoteIterators.fromSingleton(toLocatedFileStatus(fileStatus));
+ }
+ }
+ return subFiles;
+ }
+
+ private RawLocatedFileStatus toLocatedFileStatus(RawFileStatus status) throws IOException {
+ return new RawLocatedFileStatus(status,
+ status.isFile() ? getFileBlockLocations(status, 0, status.getLen()) : null);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(
+ Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flag,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ Path qualified = makeQualified(path);
+ return create(qualified, permission, flag.contains(CreateFlag.OVERWRITE),
+ bufferSize, replication, blockSize, progress);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException {
+ throw new IOException("Not supported");
+ }
+
+ /**
+ * Rename src path to dest path, if dest path is an existed dir,
+ * then FS will rename the src path under the dst dir.
+ * E.g. rename('/a/b', '/a/c') and dest 'c' is an existed dir,
+ * then the source path '/a/b' will be renamed with dest path '/a/b/c' internally.
+ *
+ * <ul>
+ * <li>Return false if src doesn't exist</li>
+ * <li>Return false if src is root</li>
+ * <li>Return false if dst path is under src path, e.g. rename('/a/b', '/a/b/c')</li>
+ * <li>Return false if dst path already exists</li>
+ * <li>Return true if rename('/a/b', '/a/b') and 'b' is an existed file</li>
+ * <li>Return true if rename('/a/b', '/a') and 'a' is an existed dir,
+ * fs will rename '/a/b' to '/a/b' internally</li>
+ * <li>Return false if rename('/a/b', '/a/b') and 'b' is an existed dir,
+ * because fs will try to rename '/a/b' to '/a/b/b', which is under '/a/b', this behavior is forbidden.</li>
+ * </ul>
+ *
+ * @param src path to be renamed
+ * @param dst path after rename
+ * @return true if rename is successful
+ * @throws IOException on failure
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ LOG.debug("Rename source path {} to dest path {}", src, dst);
+
+ // 1. Check source and destination path
+ Future<FileStatus> srcStatusFuture = taskThreadPool.submit(() -> checkAndGetSrcStatus(src));
+ Future<Path> destPathFuture = taskThreadPool.submit(() -> checkAndGetDstPath(src, dst));
+
+ FileStatus srcStatus;
+ Path dstPath;
+ try {
+ srcStatus = srcStatusFuture.get();
+ dstPath = destPathFuture.get();
+
+ if (src.equals(dstPath)) {
+ return true;
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ LOG.error("Failed to rename path, src: {}, dst: {}", src, dst, e);
+ return false;
+ }
+
+ // 2. Start copy source to destination
+ if (srcStatus.isDirectory()) {
+ fsOps.renameDir(srcStatus.getPath(), dstPath);
+ } else {
+ fsOps.renameFile(srcStatus.getPath(), dstPath, srcStatus.getLen());
+ }
+
+ return true;
+ }
+
+ private Path checkAndGetDstPath(Path src, Path dest) throws IOException {
+ FileStatus destStatus = getFileStatusOrNull(dest);
+ // 1. Rebuilding the destination path
+ Path finalDstPath = dest;
+ if (destStatus != null && destStatus.isDirectory()) {
+ finalDstPath = new Path(dest, src.getName());
+ }
+
+ // 2. No need to check the dest path because renaming itself is allowed.
+ if (src.equals(finalDstPath)) {
+ return finalDstPath;
+ }
+
+ // 3. Ensure the source path cannot be the ancestor of destination path.
+ if (RawFSUtils.inSubtree(src, finalDstPath)) {
+ throw new IOException(String.format("Failed to rename since it is prohibited to " +
+ "rename dest path %s under src path %s", finalDstPath, src));
+ }
+
+ // 4. Ensure the destination path doesn't exist.
+ FileStatus finalDstStatus = destStatus;
+ if (destStatus != null && destStatus.isDirectory()) {
+ finalDstStatus = getFileStatusOrNull(finalDstPath);
+ }
+ if (finalDstStatus != null) {
+ throw new FileAlreadyExistsException(
+ String.format("Failed to rename since the dest path %s already exists.", finalDstPath));
+ } else {
+ return finalDstPath;
+ }
+ }
+
+ private FileStatus checkAndGetSrcStatus(Path src) throws IOException {
+ // throw FileNotFoundException if src not found.
+ FileStatus srcStatus = innerFileStatus(src);
+
+ if (src.isRoot()) {
+ throw new IOException(String.format("Cannot rename the root directory %s to another name", src));
+ }
+ return srcStatus;
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive) throws IOException {
+ LOG.debug("Delete path {} - recursive {}", f, recursive);
+ try {
+ FileStatus fileStatus = getFileStatus(f);
+ Path path = fileStatus.getPath();
+
+ if (path.isRoot()) {
+ return deleteRoot(path, recursive);
+ } else {
+ if (fileStatus.isDirectory()) {
+ fsOps.deleteDir(path, recursive);
+ } else {
+ fsOps.deleteFile(path);
+ }
+ return true;
+ }
+ } catch (FileNotFoundException e) {
+ LOG.debug("Couldn't delete {} - does not exist", f);
+ return false;
+ }
+ }
+
+ /**
+ * Reject deleting root directory and implement the specific logic to compatible with
+ * AbstractContractRootDirectoryTest rm test cases.
+ *
+ * @param root the root path.
+ * @param recursive indicate whether delete directory recursively
+ * @return true if root directory is empty, false if trying to delete a non-empty dir recursively.
+ * @throws IOException if trying to delete the non-empty root dir non-recursively.
+ */
+ private boolean deleteRoot(Path root, boolean recursive) throws IOException {
+ LOG.info("Delete the {} root directory of {}", bucket, recursive);
+ boolean isEmptyDir = fsOps.isEmptyDirectory(root);
+ if (isEmptyDir) {
+ return true;
+ }
+ if (recursive) {
+ // AbstractContractRootDirectoryTest#testRmRootRecursive doesn't expect any exception if trying to delete a
+ // non-empty root directory recursively, so we have to return false here instead of throwing a IOException.
+ return false;
+ } else {
+ // AbstractContractRootDirectoryTest#testRmNonEmptyRootDirNonRecursive expect a exception if trying to delete a
+ // non-empty root directory non-recursively, so we have to throw a IOException instead of returning false.
+ throw new PathIOException(bucket, "Cannot delete root path");
+ }
+ }
+
+ @Override
+ public RawFileStatus[] listStatus(Path f) throws IOException {
+ LOG.debug("List status for path: {}", f);
+ return Iterators.toArray(listStatus(f, false), RawFileStatus.class);
+ }
+
+ public Iterator<RawFileStatus> listStatus(Path f, boolean recursive) throws IOException {
+ Path path = makeQualified(f);
+ // Assuming path is a dir at first.
+ Iterator<RawFileStatus> iterator = fsOps.listDir(path, recursive, key -> true).iterator();
+ if (iterator.hasNext()) {
+ return iterator;
+ } else {
+ RawFileStatus fileStatus = innerFileStatus(path);
+ if (fileStatus.isFile()) {
+ return Collections.singletonList(fileStatus).iterator();
+ } else {
+ // The path is an empty dir.
+ return Collections.emptyIterator();
+ }
+ }
+ }
+
+ @Override
+ public RemoteIterator<FileStatus> listStatusIterator(Path p) throws IOException {
+ // We expect throw FileNotFoundException if the path doesn't exist during creating the RemoteIterator instead of
+ // throwing FileNotFoundException during call hasNext method.
+
+ // The follow RemoteIterator is as same as {@link FileSystem#DirListingIterator} above hadoop 3.2.2,
+ // but below 3.2.2, the DirListingIterator fetches the directory entries during call hasNext method instead of
+ // create the DirListingIterator instance.
+ return new RemoteIterator<FileStatus>() {
+ private DirectoryEntries entries = listStatusBatch(p, null);
+ private int index = 0;
+
+ @Override
+ public boolean hasNext() {
+ return index < entries.getEntries().length || entries.hasMore();
+ }
+
+ private void fetchMore() throws IOException {
+ byte[] token = entries.getToken();
+ entries = listStatusBatch(p, token);
+ index = 0;
+ }
+
+ @Override
+ public FileStatus next() throws IOException {
+ if (!hasNext()) {
+ throw new NoSuchElementException("No more items in iterator");
+ } else {
+ if (index == entries.getEntries().length) {
+ fetchMore();
+ if (!hasNext()) {
+ throw new NoSuchElementException("No more items in iterator");
+ }
+ }
+
+ return entries.getEntries()[index++];
+ }
+ }
+ };
+ }
+
+ public static long dateToLong(final Date date) {
+ return date == null ? 0L : date.getTime();
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ @Override
+ public void setWorkingDirectory(Path new_dir) {
+ this.workingDir = new_dir;
+ }
+
+ @Override
+ public boolean mkdirs(Path path, FsPermission permission) throws IOException {
+ try {
+ FileStatus fileStatus = innerFileStatus(path);
+ if (fileStatus.isDirectory()) {
+ return true;
+ } else {
+ throw new FileAlreadyExistsException("Path is a file: " + path);
+ }
+ } catch (FileNotFoundException e) {
+ Path dir = makeQualified(path);
+ validatePath(dir);
+ fsOps.mkdirs(dir);
+ }
+ return true;
+ }
+
+ private void validatePath(Path path) throws IOException {
+ Path parent = path.getParent();
+ do {
+ try {
+ FileStatus fileStatus = innerFileStatus(parent);
+ if (fileStatus.isDirectory()) {
+ // If path exists and a directory, exit
+ break;
+ } else {
+ throw new FileAlreadyExistsException(String.format("Can't make directory for path '%s', it is a file.",
+ parent));
+ }
+ } catch (FileNotFoundException ignored) {
+ }
+ parent = parent.getParent();
+ } while (parent != null);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path path) throws IOException {
+ try {
+ return innerFileStatus(path);
+ } catch (ParentNotDirectoryException e) {
+ // Treat ParentNotDirectoryException as FileNotFoundException for the case that check whether path exist or not.
+ throw new FileNotFoundException(e.getMessage());
+ }
+ }
+
+
+ /**
+ * Get the file status of given path.
+ *
+ * @param f the path
+ * @return {@link RawFileStatus} describe file status info.
+ * @throws FileNotFoundException if the path doesn't exist.
+ * @throws ParentNotDirectoryException if the path is locating under an existing file, which is not allowed
+ * in directory bucket case.
+ */
+ RawFileStatus innerFileStatus(Path f) throws ParentNotDirectoryException, FileNotFoundException {
+ Path qualifiedPath = f.makeQualified(uri, workingDir);
+ RawFileStatus fileStatus = getFileStatusOrNull(qualifiedPath);
+ if (fileStatus == null) {
+ throw new FileNotFoundException(String.format("No such file or directory: %s", qualifiedPath));
+ }
+ return fileStatus;
+ }
+
+ /**
+ * The different with {@link RawFileSystem#getFileStatus(Path)} is that:
+ * 1. throw {@link ParentNotDirectoryException} if the path is locating under an existing file in directory bucket
+ * case, but {@link RawFileSystem#getFileStatus(Path)} will ignore whether the invalid path and
+ * throw {@link FileNotFoundException}
+ * 2. return null if the path doesn't exist instead of throwing {@link FileNotFoundException}.
+ *
+ * @param path the object path.
+ * @return null if the path doesn't exist.
+ * @throws ParentNotDirectoryException if the path is locating under an existing file, which is not allowed
+ * in directory bucket case.
+ */
+ public RawFileStatus getFileStatusOrNull(final Path path) throws ParentNotDirectoryException {
+ Path qualifiedPath = path.makeQualified(uri, workingDir);
+ String key = ObjectUtils.pathToKey(qualifiedPath);
+
+ // Root directory always exists
+ if (key.isEmpty()) {
+ return new RawFileStatus(0, true, 0, 0, qualifiedPath, username, Constants.MAGIC_CHECKSUM);
+ }
+
+ try {
+ ObjectInfo obj = storage.objectStatus(key);
+ if (obj == null) {
+ return null;
+ } else {
+ return objectToFileStatus(obj);
+ }
+ } catch (InvalidObjectKeyException e) {
+ String msg = String.format("The object key %s is a invalid key, detail: %s", key, e.getMessage());
+ throw new ParentNotDirectoryException(msg);
+ }
+ }
+
+ private RawFileStatus objectToFileStatus(ObjectInfo obj) {
+ Path keyPath = makeQualified(ObjectUtils.keyToPath(obj.key()));
+ long blockSize = obj.isDir() ? 0 : getDefaultBlockSize(keyPath);
+ long modificationTime = dateToLong(obj.mtime());
+ return new RawFileStatus(obj.size(), obj.isDir(), blockSize, modificationTime, keyPath, username, obj.checksum());
+ }
+
+ @Override
+ @Deprecated
+ public long getDefaultBlockSize() {
+ return getConf().getLongBytes(DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE_DEFAULT);
+ }
+
+ @Override
+ public FsServerDefaults getServerDefaults(Path p) {
+ Configuration config = getConf();
+ // CRC32 is chosen as default as it is available in all
+ // releases that support checksum.
+ // The client trash configuration is ignored.
+ return new FsServerDefaults(getDefaultBlockSize(),
+ config.getInt("dfs.bytes-per-checksum", 512),
+ 64 * 1024,
+ getDefaultReplication(),
+ config.getInt(CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
+ CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT),
+ false,
+ CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT,
+ DataChecksum.Type.CRC32,
+ "");
+ }
+
+ private void stopAllServices() {
+ ThreadPools.shutdown(uploadThreadPool, 30, TimeUnit.SECONDS);
+ ThreadPools.shutdown(taskThreadPool, 30, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void initialize(URI uri, Configuration conf) throws IOException {
+ super.initialize(uri, conf);
+ setConf(conf);
+ this.scheme = FSUtils.scheme(conf, uri);
+
+ // Username is the current user at the time the FS was instantiated.
+ this.username = UserGroupInformation.getCurrentUser().getShortUserName();
+ this.workingDir = new Path("/user", username).makeQualified(uri, null);
+ this.uri = URI.create(scheme + "://" + uri.getAuthority());
+ this.bucket = this.uri.getAuthority();
+ this.storage = ObjectStorageFactory.create(scheme, bucket, getConf());
+ if (storage.bucket() == null) {
+ throw new FileNotFoundException(String.format("Bucket: %s not found.", uri.getAuthority()));
+ }
+
+ int taskThreadPoolSize = getConf().getInt(ConfKeys.FS_TASK_THREAD_POOL_SIZE.key(storage.scheme()),
+ ConfKeys.FS_TASK_THREAD_POOL_SIZE_DEFAULT);
+ this.taskThreadPool = ThreadPools.newWorkerPool(TASK_THREAD_POOL_PREFIX, taskThreadPoolSize);
+
+ int uploadThreadPoolSize =
+ getConf().getInt(ConfKeys.FS_MULTIPART_THREAD_POOL_SIZE.key(storage.scheme()),
+ ConfKeys.FS_MULTIPART_THREAD_POOL_SIZE_DEFAULT);
+ this.uploadThreadPool = ThreadPools.newWorkerPool(MULTIPART_THREAD_POOL_PREFIX, uploadThreadPoolSize);
+
+ if (storage.bucket().isDirectory()) {
+ fsOps = new DirectoryFsOps((DirectoryStorage) storage, this::objectToFileStatus);
+ } else {
+ fsOps = new DefaultFsOps(storage, getConf(), taskThreadPool, this::objectToFileStatus);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ super.close();
+ storage.close();
+ } finally {
+ stopAllServices();
+ }
+ }
+
+ public ObjectStorage storage() {
+ return storage;
+ }
+
+ public ExecutorService uploadThreadPool() {
+ return uploadThreadPool;
+ }
+
+ /**
+ * @return null if checksum is not supported.
+ */
+ @Override
+ public FileChecksum getFileChecksum(Path f, long length) throws IOException {
+ Preconditions.checkArgument(length >= 0);
+
+ RawFileStatus fileStatus = innerFileStatus(f);
+ if (fileStatus.isDirectory()) {
+ // Compatible with HDFS
+ throw new FileNotFoundException(String.format("Path is not a file, %s", f));
+ }
+ if (!getConf().getBoolean(ConfKeys.FS_CHECKSUM_ENABLED.key(storage.scheme()),
+ ConfKeys.FS_CHECKSUM_ENABLED_DEFAULT)) {
+ return null;
+ }
+
+ ChecksumInfo csInfo = storage.checksumInfo();
+ return new TosChecksum(csInfo.algorithm(), fileStatus.checksum());
+ }
+
+ @Override
+ public String getCanonicalServiceName() {
+ return null;
+ }
+
+ @Override
+ public void setXAttr(Path path, String name, byte[] value, EnumSet<XAttrSetFlag> flag) throws IOException {
+ Preconditions.checkNotNull(name, "xAttr name must not be null.");
+ Preconditions.checkArgument(!name.isEmpty(), "xAttr name must not be empty.");
+ Preconditions.checkNotNull(value, "xAttr value must not be null.");
+
+ if (getFileStatus(path).isFile()) {
+ Path qualifiedPath = path.makeQualified(uri, workingDir);
+ String key = ObjectUtils.pathToKey(qualifiedPath);
+
+ Map<String, String> existedTags = storage.getTags(key);
+ validateXAttrFlag(name, existedTags.containsKey(name), flag);
+
+ String newValue = Bytes.toString(value);
+ String previousValue = existedTags.put(name, newValue);
+ if (!newValue.equals(previousValue)) {
+ storage.putTags(key, existedTags);
+ }
+ }
+ }
+
+ @Override
+ public Map<String, byte[]> getXAttrs(Path path) throws IOException {
+ if (getFileStatus(path).isDirectory()) {
+ return new HashMap<>();
+ } else {
+ Path qualifiedPath = path.makeQualified(uri, workingDir);
+ String key = ObjectUtils.pathToKey(qualifiedPath);
+
+ Map<String, String> tags = storage.getTags(key);
+ return tags.entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, t -> Bytes.toBytes(t.getValue())));
+ }
+ }
+
+ @Override
+ public byte[] getXAttr(Path path, String name) throws IOException {
+ Map<String, byte[]> xAttrs = getXAttrs(path);
+ if (xAttrs.containsKey(name)) {
+ return xAttrs.get(name);
+ } else {
+ throw new IOException("Attribute with name " + name + " is not found.");
+ }
+ }
+
+ @Override
+ public Map<String, byte[]> getXAttrs(Path path, List<String> names) throws IOException {
+ Map<String, byte[]> xAttrs = getXAttrs(path);
+ xAttrs.keySet().retainAll(names);
+ if (xAttrs.size() == names.size()) {
+ return xAttrs;
+ } else {
+ List<String> badNames = names.stream().filter(n -> !xAttrs.containsKey(n)).collect(Collectors.toList());
+ throw new IOException("Attributes with name " + badNames + " are not found.");
+ }
+ }
+
+ @Override
+ public List<String> listXAttrs(Path path) throws IOException {
+ return Lists.newArrayList(getXAttrs(path).keySet());
+ }
+
+ @Override
+ public void removeXAttr(Path path, String name) throws IOException {
+ if (getFileStatus(path).isFile()) {
+ Path qualifiedPath = path.makeQualified(uri, workingDir);
+ String key = ObjectUtils.pathToKey(qualifiedPath);
+
+ Map<String, String> existedTags = storage.getTags(key);
+ if (existedTags.remove(name) != null) {
+ storage.putTags(key, existedTags);
+ }
+ }
+ }
+
+ private void validateXAttrFlag(String xAttrName, boolean xAttrExists, EnumSet<XAttrSetFlag> flag) throws IOException {
+ if (xAttrExists) {
+ if (!flag.contains(REPLACE)) {
+ throw new IOException("XAttr: " + xAttrName + " already exists. The REPLACE flag must be specified.");
+ }
+ } else {
+ if (!flag.contains(CREATE)) {
+ throw new IOException("XAttr: " + xAttrName + " does not exist. The CREATE flag must be specified.");
+ }
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawLocatedFileStatus.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawLocatedFileStatus.java
new file mode 100644
index 00000000000..562289455d3
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/RawLocatedFileStatus.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.LocatedFileStatus;
+
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
+
+public class RawLocatedFileStatus extends LocatedFileStatus {
+ public RawLocatedFileStatus(RawFileStatus status, BlockLocation[] locations) {
+ super(checkNotNull(status), locations);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/TosChecksum.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/TosChecksum.java
new file mode 100644
index 00000000000..f0bedc60c4b
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/TosChecksum.java
@@ -0,0 +1,67 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.tosfs.common.Bytes;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class TosChecksum extends FileChecksum {
+ private String algorithm;
+ private byte[] checksum;
+
+ public TosChecksum(String algorithm, byte[] checksum) {
+ this.algorithm = algorithm;
+ this.checksum = checksum;
+ }
+
+ @Override
+ public String getAlgorithmName() {
+ return algorithm;
+ }
+
+ @Override
+ public int getLength() {
+ return checksum.length;
+ }
+
+ @Override
+ public byte[] getBytes() {
+ return checksum;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ byte[] algorithmBytes = Bytes.toBytes(algorithm);
+ out.write(algorithmBytes.length);
+ out.write(algorithmBytes);
+ out.write(checksum.length);
+ out.write(checksum);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ byte[] algorithmBytes = new byte[in.readInt()];
+ in.readFully(algorithmBytes);
+ algorithm = Bytes.toString(algorithmBytes);
+ checksum = new byte[in.readInt()];
+ in.readFully(checksum);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/TosFS.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/TosFS.java
new file mode 100644
index 00000000000..3146e2d6cb7
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/TosFS.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.tosfs.object.tos.TOS;
+import org.apache.hadoop.util.Preconditions;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class TosFS extends RawFS {
+ public TosFS(URI uri, Configuration conf) throws IOException, URISyntaxException {
+ super(verifyURI(uri, conf), conf);
+ }
+
+ private static URI verifyURI(URI uri, Configuration conf) {
+ Preconditions.checkNotNull(uri);
+
+ String scheme = uri.getScheme();
+ if (scheme == null || scheme.isEmpty()) {
+ scheme = FileSystem.getDefaultUri(conf).getScheme();
+ }
+ Preconditions.checkArgument(scheme.equals(TOS.TOS_SCHEME),
+ "Unsupported scheme %s, expected scheme is %s.", scheme, TOS.TOS_SCHEME);
+
+ return uri;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/TosFileSystem.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/TosFileSystem.java
new file mode 100644
index 00000000000..eeda1bc730d
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/TosFileSystem.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.tosfs.object.tos.TOS;
+import org.apache.hadoop.util.Preconditions;
+
+import java.io.IOException;
+import java.net.URI;
+
+public class TosFileSystem extends RawFileSystem {
+ @Override
+ public void initialize(URI uri, Configuration conf) throws IOException {
+ Preconditions.checkNotNull(uri);
+
+ String scheme = uri.getScheme();
+ if (scheme == null || scheme.isEmpty()) {
+ scheme = FileSystem.getDefaultUri(conf).getScheme();
+ }
+ Preconditions.checkArgument(scheme.equals(TOS.TOS_SCHEME),
+ "Unsupported scheme %s, expected scheme is %s.", scheme, TOS.TOS_SCHEME);
+
+ super.initialize(uri, conf);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitContext.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitContext.java
new file mode 100644
index 00000000000..40d371bcf88
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitContext.java
@@ -0,0 +1,45 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.util.Lists;
+
+import java.util.List;
+
+public class CommitContext {
+ private final List<FileStatus> pendingSets;
+ // It will be accessed in multi-threads, please access it in a thread-safe context.
+ private final List<String> destKeys;
+
+ public CommitContext(List<FileStatus> pendingSets) {
+ this.pendingSets = pendingSets;
+ this.destKeys = Lists.newArrayList();
+ }
+
+ public List<FileStatus> pendingSets() {
+ return pendingSets;
+ }
+
+ public synchronized void addDestKey(String destKey) {
+ destKeys.add(destKey);
+ }
+
+ public synchronized List<String> destKeys() {
+ return destKeys;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitUtils.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitUtils.java
new file mode 100644
index 00000000000..f3a9f6c301e
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitUtils.java
@@ -0,0 +1,359 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.tosfs.commit.mapred.Committer;
+import org.apache.hadoop.fs.tosfs.util.Serializer;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Supplier;
+
+public class CommitUtils {
+ private CommitUtils() {
+ }
+
+ public static final String COMMITTER_NAME = Committer.class.getName();
+
+ /**
+ * Support scheme for tos committer.
+ */
+ public static final String FS_STORAGE_OBJECT_SCHEME = "fs.object-storage.scheme";
+ public static final String DEFAULT_FS_STORAGE_OBJECT_SCHEME = "tos,oss,s3,s3a,s3n,obs,filestore";
+
+ /**
+ * Path for "magic" writes: path and {@link #PENDING_SUFFIX} files: {@value}.
+ */
+ public static final String MAGIC = "__magic";
+
+ /**
+ * Marker of the start of a directory tree for calculating the final path names: {@value}.
+ */
+ public static final String BASE = "__base";
+
+ /**
+ * Suffix applied to pending commit metadata: {@value}.
+ */
+ public static final String PENDING_SUFFIX = ".pending";
+
+ /**
+ * Suffix applied to multiple pending commit metadata: {@value}.
+ */
+ public static final String PENDINGSET_SUFFIX = ".pendingset";
+
+ /**
+ * Marker file to create on success: {@value}.
+ */
+ public static final String _SUCCESS = "_SUCCESS";
+
+ /**
+ * Format string used to build a summary file from a Job ID.
+ */
+ public static final String SUMMARY_FILENAME_FORMAT = "summary-%s.json";
+
+ /**
+ * Extra Data key for task attempt in pendingset files.
+ */
+ public static final String TASK_ATTEMPT_ID = "task.attempt.id";
+
+ /**
+ * The UUID for jobs: {@value}.
+ * This was historically created in Spark 1.x's SQL queries, see SPARK-33230.
+ */
+ public static final String SPARK_WRITE_UUID = "spark.sql.sources.writeJobUUID";
+
+ /**
+ * Get the magic location for the output path.
+ * Format: ${out}/__magic
+ *
+ * @param out the base output directory.
+ * @return the location of magic job attempts.
+ */
+ public static Path magicPath(Path out) {
+ return new Path(out, MAGIC);
+ }
+
+ /**
+ * Compute the "magic" path for a job. <br>
+ * Format: ${jobOutput}/__magic/${jobId}
+ *
+ * @param jobId unique Job ID.
+ * @param jobOutput the final output directory.
+ * @return the path to store job attempt data.
+ */
+ public static Path magicJobPath(String jobId, Path jobOutput) {
+ return new Path(magicPath(jobOutput), jobId);
+ }
+
+ /**
+ * Get the Application Attempt ID for this job.
+ *
+ * @param context the context to look in
+ * @return the Application Attempt ID for a given job, or 0
+ */
+ public static int appAttemptId(JobContext context) {
+ return context.getConfiguration().getInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 0);
+ }
+
+ /**
+ * Compute the "magic" path for a job attempt. <br>
+ * Format: ${jobOutput}/__magic/${jobId}/${appAttemptId}
+ *
+ * @param jobId unique Job ID.
+ * @param appAttemptId the ID of the application attempt for this job.
+ * @param jobOutput the final output directory.
+ * @return the path to store job attempt data.
+ */
+ public static Path magicJobAttemptPath(String jobId, int appAttemptId, Path jobOutput) {
+ return new Path(magicPath(jobOutput), formatAppAttemptDir(jobId, appAttemptId));
+ }
+
+ /**
+ * Compute the "magic" path for a job attempt. <br>
+ * Format: ${jobOutput}/__magic/${jobId}/${appAttemptId}
+ *
+ * @param context the context of the job.
+ * @param jobOutput the final output directory.
+ * @return the path to store job attempt data.
+ */
+ public static Path magicJobAttemptPath(JobContext context, Path jobOutput) {
+ String jobId = buildJobId(context);
+ return magicJobAttemptPath(jobId, appAttemptId(context), jobOutput);
+ }
+
+ private static String formatAppAttemptDir(String jobId, int appAttemptId) {
+ return String.format("%s/%02d", jobId, appAttemptId);
+ }
+
+ /**
+ * Compute the path where the output of magic task attempts are stored. <br>
+ * Format: ${jobOutput}/__magic/${jobId}/${appAttemptId}/tasks
+ *
+ * @param jobId unique Job ID.
+ * @param jobOutput The output path to commit work into.
+ * @param appAttemptId the ID of the application attempt for this job.
+ * @return the path where the output of magic task attempts are stored.
+ */
+ public static Path magicTaskAttemptsPath(String jobId, Path jobOutput, int appAttemptId) {
+ return new Path(magicJobAttemptPath(jobId, appAttemptId, jobOutput), "tasks");
+ }
+
+ /**
+ * Compute the path where the output of a task attempt is stored until that task is committed.
+ * This path is marked as a base path for relocations, so subdirectory information is preserved.
+ * Format: ${jobOutput}/__magic/${jobId}/${appAttemptId}/tasks/${taskAttemptId}/__base
+ *
+ * @param context the context of the task attempt.
+ * @param jobId unique Job ID.
+ * @param jobOutput The output path to commit work into.
+ * @return the path where a task attempt should be stored.
+ */
+ public static Path magicTaskAttemptBasePath(TaskAttemptContext context, String jobId, Path jobOutput) {
+ return new Path(magicTaskAttemptPath(context, jobId, jobOutput), BASE);
+ }
+
+ /**
+ * Compute the path where the output of a task attempt is stored until that task is committed.
+ * This path is marked as a base path for relocations, so subdirectory information is preserved.
+ * Format: ${jobOutput}/__magic/${jobId}/${appAttemptId}/tasks/${taskAttemptId}/__base
+ *
+ * @param context the context of the task attempt.
+ * @param jobOutput The output path to commit work into.
+ * @return the path where a task attempt should be stored.
+ */
+ public static Path magicTaskAttemptBasePath(TaskAttemptContext context, Path jobOutput) {
+ String jobId = buildJobId(context);
+ return magicTaskAttemptBasePath(context, jobId, jobOutput);
+ }
+
+ /**
+ * Get the magic task attempt path, without any annotations to mark relative references.
+ * If there is an app attempt property in the context configuration, that is included.
+ * Format: ${jobOutput}/__magic/${jobId}/${appAttemptId}/tasks/${taskAttemptId}
+ *
+ * @param context the context of the task attempt.
+ * @param jobId unique Job ID.
+ * @param jobOutput The output path to commit work into.
+ * @return the path under which all attempts go.
+ */
+ public static Path magicTaskAttemptPath(TaskAttemptContext context, String jobId, Path jobOutput) {
+ return new Path(
+ magicTaskAttemptsPath(jobId, jobOutput, appAttemptId(context)),
+ String.valueOf(context.getTaskAttemptID()));
+ }
+
+ /**
+ * Get the magic task attempt path, without any annotations to mark relative references.
+ * If there is an app attempt property in the context configuration, that is included.
+ * Format: ${jobOutput}/__magic/${jobId}/${appAttemptId}/tasks/${taskAttemptId}
+ *
+ * @param context the context of the task attempt.
+ * @param jobOutput The output path to commit work into.
+ * @return the path under which all attempts go.
+ */
+ public static Path magicTaskAttemptPath(TaskAttemptContext context, Path jobOutput) {
+ String jobId = buildJobId(context);
+ return magicTaskAttemptPath(context, jobId, jobOutput);
+ }
+
+ /**
+ * Get the magic task pendingset path.
+ * Format: ${jobOutput}/__magic/${jobId}/${appAttemptId}/${taskId}.pendingset
+ *
+ * @param context the context of the task attempt.
+ * @param jobOutput The output path to commit work into.
+ */
+ public static Path magicTaskPendingSetPath(TaskAttemptContext context, Path jobOutput) {
+ String taskId = String.valueOf(context.getTaskAttemptID().getTaskID());
+ return new Path(magicJobAttemptPath(context, jobOutput), String.format("%s%s", taskId, PENDINGSET_SUFFIX));
+ }
+
+ public static String buildJobId(Configuration conf, JobID jobId) {
+ String jobUUID = conf.getTrimmed(SPARK_WRITE_UUID, "");
+ if (!jobUUID.isEmpty()) {
+ if (jobUUID.startsWith(JobID.JOB)) {
+ return jobUUID;
+ } else {
+ return String.format("%s_%s", JobID.JOB, jobUUID);
+ }
+ }
+
+ // if no other option was supplied, return the job ID.
+ // This is exactly what MR jobs expect, but is not what
+ // Spark jobs can do as there is a risk of jobID collision.
+ return jobId != null ? jobId.toString() : "NULL_JOB_ID";
+ }
+
+ public static String buildJobId(JobContext context) {
+ return buildJobId(context.getConfiguration(), context.getJobID());
+ }
+
+ /**
+ * Get a job name; returns meaningful text if there is no name.
+ *
+ * @param context job context
+ * @return a string for logs
+ */
+ public static String jobName(JobContext context) {
+ String name = context.getJobName();
+ return (name != null && !name.isEmpty()) ? name : "(anonymous)";
+ }
+
+ /**
+ * Format: ${output}/_SUCCESS
+ */
+ public static Path successMarker(Path output) {
+ return new Path(output, _SUCCESS);
+ }
+
+ /**
+ * Format: ${reportDir}/summary-xxxxx.json
+ */
+ public static Path summaryReport(Path reportDir, String jobId) {
+ return new Path(reportDir, String.format(SUMMARY_FILENAME_FORMAT, jobId));
+ }
+
+ public static void save(FileSystem fs, Path path, byte[] data) throws IOException {
+ // By default, fs.create(path) will create parent folder recursively, and overwrite
+ // it if it's already exist.
+ try (FSDataOutputStream out = fs.create(path)) {
+ IOUtils.copy(new ByteArrayInputStream(data), out);
+ }
+ }
+
+ public static void save(FileSystem fs, Path path, Serializer instance) throws IOException {
+ save(fs, path, instance.serialize());
+ }
+
+ public static byte[] load(FileSystem fs, Path path) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (FSDataInputStream in = fs.open(path)) {
+ IOUtils.copy(in, out);
+ }
+ return out.toByteArray();
+ }
+
+ public static List<FileStatus> listPendingFiles(FileSystem fs, Path dir) throws IOException {
+ List<FileStatus> pendingFiles = Lists.newArrayList();
+ CommitUtils.listFiles(fs, dir, true, f -> {
+ if (f.getPath().toString().endsWith(CommitUtils.PENDING_SUFFIX)) {
+ pendingFiles.add(f);
+ }
+ });
+ return pendingFiles;
+ }
+
+ public static void listFiles(FileSystem fs, Path dir, boolean recursive, FileVisitor visitor) throws IOException {
+ RemoteIterator<LocatedFileStatus> iter = fs.listFiles(dir, recursive);
+ while (iter.hasNext()) {
+ FileStatus f = iter.next();
+ visitor.visit(f);
+ }
+ }
+
+ public interface FileVisitor {
+ void visit(FileStatus f);
+ }
+
+ public static boolean supportObjectStorageCommit(Configuration conf, Path outputPath) {
+ return supportSchemes(conf).contains(outputPath.toUri().getScheme());
+ }
+
+ private static List<String> supportSchemes(Configuration conf) {
+ String schemes = conf.get(FS_STORAGE_OBJECT_SCHEME, DEFAULT_FS_STORAGE_OBJECT_SCHEME);
+ Preconditions.checkNotNull(schemes, "%s cannot be null", FS_STORAGE_OBJECT_SCHEME);
+ return Arrays.asList(schemes.split(","));
+ }
+
+ private static Set<String> errorStage = new HashSet<>();
+ private static boolean testMode = false;
+
+ public static void injectError(String stage) {
+ errorStage.add(stage);
+ testMode = true;
+ }
+
+ public static void removeError(String stage) {
+ errorStage.remove(stage);
+ }
+
+ public static <T extends Exception> void triggerError(Supplier<T> error, String stage) throws T {
+ if (testMode && errorStage.contains(stage)) {
+ throw error.get();
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/Committer.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/Committer.java
new file mode 100644
index 00000000000..6d016609762
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/Committer.java
@@ -0,0 +1,499 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.commit.ops.PendingOps;
+import org.apache.hadoop.fs.tosfs.commit.ops.PendingOpsFactory;
+import org.apache.hadoop.fs.tosfs.common.Tasks;
+import org.apache.hadoop.fs.tosfs.common.ThreadPools;
+import org.apache.hadoop.fs.tosfs.object.MultipartUpload;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
+import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+public class Committer extends PathOutputCommitter {
+ private static final Logger LOG = LoggerFactory.getLogger(Committer.class);
+
+ public static final String COMMITTER_THREADS = "fs.job.committer.threads";
+ public static final String COMMITTER_SUMMARY_REPORT_DIR = "fs.job.committer.summary.report.directory";
+ public static final int DEFAULT_COMMITTER_THREADS = Runtime.getRuntime().availableProcessors();
+ public static final String THREADS_PREFIX = "job-committer-thread-pool";
+
+ private final String jobId;
+ private final Path outputPath;
+ // This is the directory for all intermediate work, where the output format will write data.
+ // This may not be on the final file system
+ private Path workPath;
+ private final String role;
+ private final Configuration conf;
+ private final FileSystem destFs;
+ private final ObjectStorage storage;
+ private final PendingOps ops;
+
+ public Committer(Path outputPath, TaskAttemptContext context) throws IOException {
+ this(outputPath, context, String.format("Task committer %s", context.getTaskAttemptID()));
+ this.workPath = CommitUtils.magicTaskAttemptBasePath(context, outputPath);
+ LOG.info("Task attempt {} has work path {}", context.getTaskAttemptID(), getWorkPath());
+ }
+
+ public Committer(Path outputPath, JobContext context) throws IOException {
+ this(outputPath, context, String.format("Job committer %s", context.getJobID()));
+ }
+
+ private Committer(Path outputPath, JobContext context, String role) throws IOException {
+ super(outputPath, context);
+ this.jobId = CommitUtils.buildJobId(context);
+ this.outputPath = outputPath;
+ this.role = role;
+ this.conf = context.getConfiguration();
+ this.destFs = outputPath.getFileSystem(conf);
+ LOG.info("{} instantiated for job '{}' ID {} with destination {}",
+ role,
+ CommitUtils.jobName(context),
+ jobId, outputPath);
+ // Initialize the object storage.
+ this.storage = ObjectStorageFactory.create(outputPath.toUri().getScheme(), outputPath.toUri().getAuthority(), conf);
+ this.ops = PendingOpsFactory.create(destFs, storage);
+ }
+
+ @Override
+ public Path getOutputPath() {
+ return outputPath;
+ }
+
+ @Override
+ public Path getWorkPath() {
+ return workPath;
+ }
+
+ @Override
+ public void setupJob(JobContext context) throws IOException {
+ checkJobId(context);
+ LOG.info("Setup Job {}", jobId);
+ Path jobOutput = getOutputPath();
+
+ // delete the success marker if exists.
+ destFs.delete(CommitUtils.successMarker(jobOutput), false);
+
+ // create the destination directory.
+ destFs.mkdirs(jobOutput);
+
+ logUncompletedMPUIfPresent(jobOutput);
+
+ // Reset the job path, and create the job path with job attempt sub path.
+ Path jobPath = CommitUtils.magicJobPath(jobId, outputPath);
+ Path jobAttemptPath = CommitUtils.magicJobAttemptPath(context, outputPath);
+ destFs.delete(jobPath, true);
+ destFs.mkdirs(jobAttemptPath);
+ }
+
+ private void logUncompletedMPUIfPresent(Path jobOutput) {
+ // do a scan and add warn log message for active uploads.
+ int nums = 0;
+ for (MultipartUpload upload : storage.listUploads(ObjectUtils.pathToKey(jobOutput, true))) {
+ if (nums++ > 10) {
+ LOG.warn("There are more than 10 uncompleted multipart uploads under path {}.", jobOutput);
+ break;
+ }
+ LOG.warn("Uncompleted multipart upload {} is under path {}, either jobs are running concurrently "
+ + "or failed jobs are not being cleaned up.", upload, jobOutput);
+ }
+ }
+
+ @Override
+ public void commitJob(JobContext context) throws IOException {
+ checkJobId(context);
+ LOG.info("{}: committing job {}", role, jobId);
+ String stage = null;
+ Exception failure = null;
+ SuccessData successData = null;
+
+ ExecutorService threadPool = ThreadPools.newWorkerPool(THREADS_PREFIX, commitThreads());
+ List<FileStatus> pendingSets = Lists.newArrayList();
+ try {
+ // Step.1 List active pending commits.
+ stage = "preparing";
+ CommitUtils.listFiles(destFs, CommitUtils.magicJobAttemptPath(context, outputPath), true, f -> {
+ if (f.getPath().toString().endsWith(CommitUtils.PENDINGSET_SUFFIX)) {
+ pendingSets.add(f);
+ }
+ });
+
+ // Step.2 Load and commit those active pending commits.
+ stage = "commit";
+ CommitContext commitCtxt = new CommitContext(pendingSets);
+ loadAndCommitPendingSets(threadPool, commitCtxt);
+
+ // Step.3 Save the success marker.
+ stage = "marker";
+ successData = createSuccessData(commitCtxt.destKeys());
+ CommitUtils.triggerError(() -> new IOException("Mock error of success marker."), stage);
+ CommitUtils.save(destFs, CommitUtils.successMarker(outputPath), successData);
+
+ // Step.4 Abort those orphan multipart uploads and cleanup the staging dir.
+ stage = "clean";
+ cleanup(threadPool, true);
+ } catch (Exception e) {
+ failure = e;
+ LOG.warn("Commit failure for job {} stage {}", CommitUtils.buildJobId(context), stage, e);
+
+ // Revert all pending sets when marker step fails.
+ if (stage.equals("marker")) {
+ CommonUtils.runQuietly(
+ () -> loadAndRevertPendingSets(threadPool, new CommitContext(pendingSets)));
+ }
+ CommonUtils.runQuietly(() -> cleanup(threadPool, true));
+ throw e;
+ } finally {
+ saveSummaryReportQuietly(stage, context, successData, failure);
+ CommonUtils.runQuietly(threadPool::shutdown);
+
+ cleanupResources();
+ }
+ }
+
+ private SuccessData createSuccessData(Iterable<String> filenames) {
+ SuccessData data = SuccessData.builder()
+ .setName(SuccessData.class.getName())
+ .setCommitter(CommitUtils.COMMITTER_NAME)
+ .setTimestamp(System.currentTimeMillis())
+ .setHostname(NetUtils.getHostname())
+ .setDescription(role)
+ .setJobId(jobId)
+ .addFileNames(filenames)
+ .build();
+
+ data.addDiagnosticInfo(COMMITTER_THREADS, Integer.toString(commitThreads()));
+ return data;
+ }
+
+ private void saveSummaryReportQuietly(String activeStage, JobContext context, SuccessData report, Throwable thrown) {
+ Configuration jobConf = context.getConfiguration();
+ String reportDir = jobConf.get(COMMITTER_SUMMARY_REPORT_DIR, "");
+ if (reportDir.isEmpty()) {
+ LOG.debug("Summary directory conf: {} is not set", COMMITTER_SUMMARY_REPORT_DIR);
+ return;
+ }
+
+ Path path = CommitUtils.summaryReport(new Path(reportDir), jobId);
+ LOG.debug("Summary report path is {}", path);
+
+ try {
+ if (report == null) {
+ report = createSuccessData(null);
+ }
+ if (thrown != null) {
+ report.recordJobFailure(thrown);
+ }
+ report.addDiagnosticInfo("stage", activeStage);
+
+ CommitUtils.save(path.getFileSystem(jobConf), path, report);
+ LOG.info("Job summary saved to {}", path);
+ } catch (Exception e) {
+ LOG.warn("Failed to save summary to {}", path, e);
+ }
+ }
+
+ private void loadAndCommitPendingSets(ExecutorService outerPool, CommitContext commitContext) {
+ ExecutorService innerPool = ThreadPools.newWorkerPool("commit-pending-files-pool", commitThreads());
+ try {
+ Tasks.foreach(commitContext.pendingSets())
+ .stopOnFailure()
+ .throwFailureWhenFinished()
+ .executeWith(outerPool)
+ .abortWith(pendingSet -> loadAndAbort(innerPool, pendingSet))
+ .revertWith(pendingSet -> loadAndRevert(innerPool, pendingSet))
+ .run(pendingSet -> loadAndCommit(commitContext, innerPool, pendingSet));
+ } finally {
+ CommonUtils.runQuietly(innerPool::shutdown);
+ }
+ }
+
+ private void loadAndRevertPendingSets(ExecutorService outerPool, CommitContext commitContext) {
+ Tasks.foreach(commitContext.pendingSets())
+ .throwFailureWhenFinished()
+ .executeWith(outerPool)
+ .run(pendingSet -> loadAndRevert(outerPool, pendingSet));
+ }
+
+ /**
+ * Load {@link PendingSet} from file and abort those {@link Pending} commits.
+ */
+ private void loadAndAbort(ExecutorService pool, FileStatus pendingSetFile) {
+ PendingSet pendingSet = PendingSet.deserialize(destFs, pendingSetFile);
+ Tasks.foreach(pendingSet.commits())
+ .suppressFailureWhenFinished()
+ .executeWith(pool)
+ .run(ops::abort);
+ }
+
+ /**
+ * Load {@link PendingSet} from file and revert those {@link Pending} commits.
+ */
+ private void loadAndRevert(ExecutorService pool, FileStatus pendingSetFile) {
+ PendingSet pendingSet = PendingSet.deserialize(destFs, pendingSetFile);
+ Tasks.foreach(pendingSet.commits())
+ .suppressFailureWhenFinished()
+ .executeWith(pool)
+ .run(ops::revert);
+ }
+
+ /**
+ * Load {@link PendingSet} from file and commit those {@link Pending} commits.
+ */
+ private void loadAndCommit(CommitContext commitCtxt, ExecutorService pool, FileStatus pendingSetFile) {
+ PendingSet pendingSet = PendingSet.deserialize(destFs, pendingSetFile);
+ // Verify that whether the job id is matched.
+ String jobId = pendingSet.jobId();
+ if (!StringUtils.isNoneEmpty(jobId) && !Objects.equals(jobId, jobId())) {
+ throw new IllegalStateException(String.format("Mismatch in Job ID (%s) and commit job ID (%s)", jobId(), jobId));
+ }
+
+ Tasks.foreach(pendingSet.commits())
+ .stopOnFailure()
+ .throwFailureWhenFinished()
+ .executeWith(pool)
+ .onFailure((pending, exception) -> ops.abort(pending))
+ .abortWith(ops::abort)
+ .revertWith(ops::revert)
+ .run(pending -> {
+ ops.commit(pending);
+ commitCtxt.addDestKey(pending.destKey());
+ });
+ }
+
+ @Override
+ public void abortJob(JobContext context, JobStatus.State state) {
+ checkJobId(context);
+ LOG.info("{}: aborting job {} in state {}", role, jobId, state);
+ ExecutorService service = ThreadPools.newWorkerPool(THREADS_PREFIX, commitThreads());
+ try {
+ cleanup(service, false);
+ } finally {
+ service.shutdown();
+
+ cleanupResources();
+ }
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext context) throws IOException {
+ checkJobId(context);
+ LOG.info("Setup Task {}", context.getTaskAttemptID());
+ Path taskAttemptBasePath = CommitUtils.magicTaskAttemptBasePath(context, outputPath);
+ // Delete the task attempt path if somehow it was there.
+ destFs.delete(taskAttemptBasePath, true);
+ // Make an empty directory.
+ destFs.mkdirs(taskAttemptBasePath);
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext taskContext) {
+ return true;
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext context) throws IOException {
+ checkJobId(context);
+ LOG.info("Commit task {}", context);
+ ExecutorService pool = ThreadPools.newWorkerPool(THREADS_PREFIX, commitThreads());
+ try {
+ PendingSet commits = innerCommitTask(pool, context);
+ LOG.info("Task {} committed {} files", context.getTaskAttemptID(), commits.size());
+ } catch (IOException e) {
+ LOG.error("Failed to commit task {}", context.getTaskAttemptID(), e);
+ throw e;
+ } finally {
+ // Shutdown the thread pool quietly.
+ CommonUtils.runQuietly(pool::shutdown);
+
+ // Delete the task attempt path quietly.
+ Path taskAttemptPath = CommitUtils.magicTaskAttemptPath(context, outputPath);
+ LOG.info("Delete task attempt path {}", taskAttemptPath);
+ CommonUtils.runQuietly(() -> destFs.delete(taskAttemptPath, true));
+ }
+ }
+
+ private PendingSet innerCommitTask(ExecutorService pool, TaskAttemptContext context) throws IOException {
+ Path taskAttemptBasePath = CommitUtils.magicTaskAttemptBasePath(context, outputPath);
+ PendingSet pendingSet = new PendingSet(jobId);
+ try {
+ // Load the pending files and fill them into the pending set.
+ List<FileStatus> pendingFiles = CommitUtils.listPendingFiles(destFs, taskAttemptBasePath);
+ // Use the thread-safe collection to collect the pending list.
+ List<Pending> pendings = Collections.synchronizedList(Lists.newArrayList());
+ Tasks.foreach(pendingFiles)
+ .throwFailureWhenFinished()
+ .executeWith(pool)
+ .run(f -> {
+ try {
+ byte[] data = CommitUtils.load(destFs, f.getPath());
+ pendings.add(Pending.deserialize(data));
+ } catch (IOException e) {
+ LOG.warn("Failed to load .pending file {}", f.getPath(), e);
+ throw new UncheckedIOException(e);
+ }
+ });
+ pendingSet.addAll(pendings);
+
+ // Add the extra task attempt id property.
+ String taskId = String.valueOf(context.getTaskAttemptID());
+ pendingSet.addExtraData(CommitUtils.TASK_ATTEMPT_ID, taskId);
+
+ // Save the pending set to file system.
+ Path taskOutput = CommitUtils.magicTaskPendingSetPath(context, outputPath);
+ LOG.info("Saving work of {} to {}", taskId, taskOutput);
+ CommitUtils.save(destFs, taskOutput, pendingSet.serialize());
+
+ } catch (Exception e) {
+ LOG.error("Encounter error when loading pending set from {}", taskAttemptBasePath, e);
+ if (!pendingSet.commits().isEmpty()) {
+ Tasks.foreach(pendingSet.commits())
+ .executeWith(pool)
+ .suppressFailureWhenFinished()
+ .run(ops::abort);
+ }
+ throw e;
+ }
+
+ return pendingSet;
+ }
+
+ @Override
+ public void abortTask(TaskAttemptContext context) throws IOException {
+ checkJobId(context);
+ Path taskAttemptBasePath = CommitUtils.magicTaskAttemptBasePath(context, outputPath);
+ try {
+ // Load the pending files from the underlying filesystem.
+ List<FileStatus> pendingFiles = CommitUtils.listPendingFiles(destFs, taskAttemptBasePath);
+ Tasks.foreach(pendingFiles)
+ .throwFailureWhenFinished()
+ .run(f -> {
+ try {
+ byte[] serializedData = CommitUtils.load(destFs, f.getPath());
+ ops.abort(Pending.deserialize(serializedData));
+ } catch (FileNotFoundException e) {
+ LOG.debug("Listed file already deleted: {}", f);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ } finally {
+ final FileStatus pendingFile = f;
+ CommonUtils.runQuietly(() -> destFs.delete(pendingFile.getPath(), false));
+ }
+ });
+ } finally {
+ CommonUtils.runQuietly(() -> destFs.delete(taskAttemptBasePath, true));
+ }
+ }
+
+ @Override
+ public void recoverTask(TaskAttemptContext context) {
+ checkJobId(context);
+ String taskId = context.getTaskAttemptID().toString();
+ throw new UnsupportedOperationException(String.format("Unable to recover task %s, output: %s", taskId, outputPath));
+ }
+
+ private int commitThreads() {
+ return conf.getInt(COMMITTER_THREADS, DEFAULT_COMMITTER_THREADS);
+ }
+
+ private void cleanup(ExecutorService pool, boolean suppress) {
+ LOG.info("Cleanup the job by abort the multipart uploads and clean staging dir, suppress {}", suppress);
+ try {
+ Path jobOutput = getOutputPath();
+ Iterable<MultipartUpload> pending =
+ storage.listUploads(ObjectUtils.pathToKey(CommitUtils.magicJobPath(jobId, jobOutput), true));
+ Tasks.foreach(pending)
+ .executeWith(pool)
+ .suppressFailureWhenFinished()
+ .run(u -> storage.abortMultipartUpload(u.key(), u.uploadId()));
+ } catch (Exception e) {
+ if (suppress) {
+ LOG.error("The following exception has been suppressed when cleanup job", e);
+ } else {
+ throw e;
+ }
+ } finally {
+ CommonUtils.runQuietly(this::cleanupStagingDir);
+ }
+ }
+
+ private void cleanupStagingDir() throws IOException {
+ // Note: different jobs share the same __magic folder, like,
+ // tos://bucket/path/to/table/__magic/job-A/..., and
+ // tos://bucket/path/to/table/__magic/job-B/...
+ // Job should only delete its own job folder to avoid the failure of other jobs,
+ // and, folder __magic should be deleted by the last job.
+ // This design does not assure the security of two jobs that one job founds there
+ // isn't another job be running, however, when it is deleting __magic but another
+ // job will visit it at the same time. We think the probability is low and we don't
+ // deal with it.
+ destFs.delete(CommitUtils.magicJobPath(jobId, outputPath), true);
+ Path magicPath = CommitUtils.magicPath(outputPath);
+ if (destFs.listStatus(magicPath).length == 0) {
+ destFs.delete(magicPath, true);
+ }
+ }
+
+ public String jobId() {
+ return jobId;
+ }
+
+ private void checkJobId(JobContext context) {
+ String jobIdInContext = CommitUtils.buildJobId(context);
+ Preconditions.checkArgument(Objects.equals(jobId, jobIdInContext),
+ String.format("JobId set in the context: %s is not consistent with the initial jobId of the committer: %s, "
+ + "please check you settings in your taskAttemptContext.", jobIdInContext, jobId));
+ }
+
+ private void cleanupResources() {
+ CommonUtils.runQuietly(storage::close);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("role", role)
+ .add("jobId", jobId)
+ .add("outputPath", outputPath)
+ .toString();
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitterFactory.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitterFactory.java
new file mode 100644
index 00000000000..d85b1b5c9af
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitterFactory.java
@@ -0,0 +1,33 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory;
+
+import java.io.IOException;
+
+public class CommitterFactory extends PathOutputCommitterFactory {
+
+ @Override
+ public PathOutputCommitter createOutputCommitter(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ return new Committer(outputPath, context);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/MagicOutputStream.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/MagicOutputStream.java
new file mode 100644
index 00000000000..eda291c1b59
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/MagicOutputStream.java
@@ -0,0 +1,146 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.object.ObjectOutputStream;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
+import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
+import org.apache.hadoop.fs.tosfs.object.Part;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.stream.Collectors;
+
+public class MagicOutputStream extends ObjectOutputStream {
+
+ private final FileSystem fs;
+ private final Path pendingPath;
+ private boolean closeStorage = false;
+
+ public MagicOutputStream(FileSystem fs, ExecutorService threadPool, Configuration conf,
+ Path magic) {
+ this(fs,
+ ObjectStorageFactory.create(magic.toUri().getScheme(), magic.toUri().getHost(), conf),
+ threadPool,
+ conf,
+ magic);
+ closeStorage = true;
+ }
+
+ public MagicOutputStream(FileSystem fs, ObjectStorage storage, ExecutorService threadPool,
+ Configuration conf, Path magic) {
+ super(storage, threadPool, conf, magic, false);
+ this.fs = fs;
+ this.pendingPath = createPendingPath(magic);
+ }
+
+ static String toDestKey(Path magicPath) {
+ Preconditions.checkArgument(isMagic(magicPath), "Destination path is not magic %s", magicPath);
+ String magicKey = ObjectUtils.pathToKey(magicPath);
+ List<String> splits = Lists.newArrayList(magicKey.split("/"));
+
+ // Break the full splits list into three collections: <parentSplits>, __magic, <childrenSplits>
+ int magicIndex = splits.indexOf(CommitUtils.MAGIC);
+ Preconditions.checkArgument(magicIndex >= 0, "Cannot locate %s in path %s", CommitUtils.MAGIC, magicPath);
+ List<String> parentSplits = splits.subList(0, magicIndex);
+ List<String> childrenSplits = splits.subList(magicIndex + 1, splits.size());
+ Preconditions.checkArgument(!childrenSplits.isEmpty(),
+ "No path found under %s for path %s", CommitUtils.MAGIC, magicPath);
+
+ // Generate the destination splits which will be joined into the destination object key.
+ List<String> destSplits = Lists.newArrayList(parentSplits);
+ if (childrenSplits.contains(CommitUtils.BASE)) {
+ // Break the <childrenDir> into three collections: <baseParentSplits>, __base, <baseChildrenSplits>, and add all
+ // <baseChildrenSplits> into the destination splits.
+ int baseIndex = childrenSplits.indexOf(CommitUtils.BASE);
+ Preconditions.checkArgument(baseIndex >= 0, "Cannot locate %s in path %s", CommitUtils.BASE, magicPath);
+ List<String> baseChildrenSplits = childrenSplits.subList(baseIndex + 1, childrenSplits.size());
+ Preconditions.checkArgument(!baseChildrenSplits.isEmpty(),
+ "No path found under %s for magic path %s", CommitUtils.BASE, magicPath);
+ destSplits.addAll(baseChildrenSplits);
+ } else {
+ // Just add the last elements of the <childrenSplits> into the destination splits.
+ String filename = childrenSplits.get(childrenSplits.size() - 1);
+ destSplits.add(filename);
+ }
+
+ return StringUtils.join(destSplits, "/");
+ }
+
+ @Override
+ protected String createDestKey(Path magicPath) {
+ return toDestKey(magicPath);
+ }
+
+ @Override
+ protected void finishUpload(String destKey, String uploadId, List<Part> parts)
+ throws IOException {
+ Pending pending = Pending.builder()
+ .setBucket(storage().bucket().name())
+ .setUploadId(uploadId)
+ .setLength(parts.stream().mapToLong(Part::size).sum())
+ .setDestKey(destKey)
+ .setCreatedTimestamp(System.currentTimeMillis())
+ .addParts(parts)
+ .build();
+
+ persist(pendingPath, pending.serialize());
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ super.close();
+ if (closeStorage) {
+ storage().close();
+ }
+ }
+
+ protected void persist(Path p, byte[] data) throws IOException {
+ CommitUtils.save(fs, p, data);
+ }
+
+ public String pendingKey() {
+ return ObjectUtils.pathToKey(pendingPath);
+ }
+
+ private static Path createPendingPath(Path magic) {
+ return new Path(magic.getParent(), String.format("%s%s", magic.getName(), CommitUtils.PENDING_SUFFIX));
+ }
+
+ // .pending and .pendingset files are not typical magic files.
+ private static boolean isInternalFile(Path p) {
+ return p.toString().endsWith(CommitUtils.PENDINGSET_SUFFIX) || p.toString().endsWith(CommitUtils.PENDING_SUFFIX);
+ }
+
+ public static boolean isMagic(Path p) {
+ Preconditions.checkNotNull(p, "path cannot be null.");
+ String path = p.toUri().getPath();
+ List<String> splits = Arrays.stream(path.split("/"))
+ .filter(StringUtils::isNoneEmpty)
+ .collect(Collectors.toList());
+ return splits.contains(CommitUtils.MAGIC) && !isInternalFile(p);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/Pending.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/Pending.java
new file mode 100644
index 00000000000..0c595dabe65
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/Pending.java
@@ -0,0 +1,181 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.tosfs.object.Part;
+import org.apache.hadoop.fs.tosfs.util.JsonCodec;
+import org.apache.hadoop.fs.tosfs.util.Serializer;
+import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Metadata that will be serialized as json and be saved in the .pending files.
+ */
+public class Pending implements Serializer {
+ private static final JsonCodec<Pending> CODEC = new JsonCodec<>(Pending.class);
+
+ private String bucket;
+ private String destKey;
+ private String uploadId;
+ private long length;
+ private long createdTimestamp;
+ private List<Part> parts;
+
+ // No-arg constructor for json serializer, don't use.
+ public Pending() {
+ }
+
+ public Pending(
+ String bucket, String destKey,
+ String uploadId, long length,
+ long createdTimestamp, List<Part> parts) {
+ this.bucket = bucket;
+ this.destKey = destKey;
+ this.uploadId = uploadId;
+ this.length = length;
+ this.createdTimestamp = createdTimestamp;
+ this.parts = parts;
+ }
+
+ public String bucket() {
+ return bucket;
+ }
+
+ public String destKey() {
+ return destKey;
+ }
+
+ public String uploadId() {
+ return uploadId;
+ }
+
+ public long length() {
+ return length;
+ }
+
+ public long createdTimestamp() {
+ return createdTimestamp;
+ }
+
+ public List<Part> parts() {
+ return parts;
+ }
+
+ @Override
+ public byte[] serialize() throws IOException {
+ return CODEC.toBytes(this);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("bucket", bucket)
+ .add("destKey", destKey)
+ .add("uploadId", uploadId)
+ .add("length", length)
+ .add("createdTimestamp", createdTimestamp)
+ .add("uploadParts", StringUtils.join(parts, ","))
+ .toString();
+ }
+
+ public static Pending deserialize(byte[] data) throws IOException {
+ return CODEC.fromBytes(data);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(bucket, destKey, uploadId, length, createdTimestamp, parts);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ } else if (!(o instanceof Pending)) {
+ return false;
+ }
+ Pending that = (Pending) o;
+ return Objects.equals(bucket, that.bucket)
+ && Objects.equals(destKey, that.destKey)
+ && Objects.equals(uploadId, that.uploadId)
+ && Objects.equals(length, that.length)
+ && Objects.equals(createdTimestamp, that.createdTimestamp)
+ && Objects.equals(parts, that.parts);
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private String bucket;
+ private String destKey;
+ private String uploadId;
+ private long length;
+ private long createdTimestamp;
+ private final List<Part> parts = Lists.newArrayList();
+
+ public Builder setBucket(String bucket) {
+ this.bucket = bucket;
+ return this;
+ }
+
+ public Builder setDestKey(String destKey) {
+ this.destKey = destKey;
+ return this;
+ }
+
+ public Builder setUploadId(String uploadId) {
+ this.uploadId = uploadId;
+ return this;
+ }
+
+ public Builder setLength(long length) {
+ this.length = length;
+ return this;
+ }
+
+ public Builder setCreatedTimestamp(long createdTimestamp) {
+ this.createdTimestamp = createdTimestamp;
+ return this;
+ }
+
+ public Builder addParts(List<Part> parts) {
+ this.parts.addAll(parts);
+ return this;
+ }
+
+ public Pending build() {
+ Preconditions.checkArgument(StringUtils.isNoneEmpty(bucket), "Empty bucket");
+ Preconditions.checkArgument(StringUtils.isNoneEmpty(destKey), "Empty object destination key");
+ Preconditions.checkArgument(StringUtils.isNoneEmpty(uploadId), "Empty uploadId");
+ Preconditions.checkArgument(length >= 0, "Invalid length: %s", length);
+ parts.forEach(part -> Preconditions.checkArgument(StringUtils.isNoneEmpty(part.eTag(), "Empty etag")));
+
+ return new Pending(
+ bucket, destKey,
+ uploadId, length,
+ createdTimestamp, parts);
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/PendingSet.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/PendingSet.java
new file mode 100644
index 00000000000..c68db9718ac
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/PendingSet.java
@@ -0,0 +1,123 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.tosfs.util.JsonCodec;
+import org.apache.hadoop.fs.tosfs.util.Serializer;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
+import org.apache.hadoop.util.Lists;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class PendingSet implements Serializer {
+ private static final JsonCodec<PendingSet> CODEC = new JsonCodec<>(PendingSet.class);
+
+ private String jobId;
+ private List<Pending> pendings;
+ private Map<String, String> extraData;
+
+ // No-arg constructor for json serializer, don't use.
+ public PendingSet() {
+ }
+
+ public PendingSet(String jobId) {
+ this(jobId, Lists.newArrayList());
+ }
+
+ public PendingSet(String jobId, List<Pending> pendings) {
+ this.jobId = jobId;
+ this.pendings = Lists.newArrayList(pendings);
+ this.extraData = Maps.newHashMap();
+ }
+
+ public PendingSet addAll(Iterable<Pending> items) {
+ Iterables.addAll(pendings, items);
+ return this;
+ }
+
+ public PendingSet add(Pending pending) {
+ pendings.add(pending);
+ return this;
+ }
+
+ public PendingSet addExtraData(String key, String val) {
+ extraData.put(key, val);
+ return this;
+ }
+
+ public String jobId() {
+ return jobId;
+ }
+
+ public List<Pending> commits() {
+ return pendings;
+ }
+
+ public Map<String, String> extraData() {
+ return extraData;
+ }
+
+ public int size() {
+ return pendings.size();
+ }
+
+ @Override
+ public byte[] serialize() throws IOException {
+ return CODEC.toBytes(this);
+ }
+
+ public static PendingSet deserialize(byte[] data) {
+ try {
+ return CODEC.fromBytes(data);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static PendingSet deserialize(FileSystem fs, FileStatus f) {
+ try {
+ return deserialize(CommitUtils.load(fs, f.getPath()));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(jobId, pendings, extraData);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ } else if (!(o instanceof PendingSet)) {
+ return false;
+ }
+ PendingSet that = (PendingSet) o;
+ return Objects.equals(jobId, that.jobId)
+ && Objects.equals(pendings, that.pendings)
+ && Objects.equals(extraData, that.extraData);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/SuccessData.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/SuccessData.java
new file mode 100644
index 00000000000..9c1e376a039
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/SuccessData.java
@@ -0,0 +1,238 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.hadoop.fs.tosfs.util.JsonCodec;
+import org.apache.hadoop.fs.tosfs.util.Serializer;
+import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
+import org.apache.hadoop.thirdparty.com.google.common.base.Throwables;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.StringUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class SuccessData implements Serializer {
+ private static final JsonCodec<SuccessData> CODEC = new JsonCodec<>(SuccessData.class);
+
+ private String name;
+ private boolean success = true;
+ private long timestamp;
+ private String date;
+ private String hostname;
+ private String committer;
+ private String description;
+ private String jobId;
+ // Filenames in the commit.
+ private final List<String> filenames = new ArrayList<>();
+
+ // Diagnostics information.
+ private final Map<String, String> diagnostics = new HashMap<>();
+
+ // No-arg constructor for json serializer, Don't use.
+ public SuccessData() {
+ }
+
+ public SuccessData(
+ String name, boolean success, long timestamp,
+ String date, String hostname, String committer,
+ String description, String jobId, List<String> filenames) {
+ this.name = name;
+ this.success = success;
+ this.timestamp = timestamp;
+ this.date = date;
+ this.hostname = hostname;
+ this.committer = committer;
+ this.description = description;
+ this.jobId = jobId;
+ this.filenames.addAll(filenames);
+ }
+
+ public String name() {
+ return name;
+ }
+
+ public boolean success() {
+ return success;
+ }
+
+ public long timestamp() {
+ return timestamp;
+ }
+
+ public String date() {
+ return date;
+ }
+
+ public String hostname() {
+ return hostname;
+ }
+
+ public String committer() {
+ return committer;
+ }
+
+ public String description() {
+ return description;
+ }
+
+ public String jobId() {
+ return jobId;
+ }
+
+ public Map<String, String> diagnostics() {
+ return diagnostics;
+ }
+
+ public List<String> filenames() {
+ return filenames;
+ }
+
+ public void recordJobFailure(Throwable thrown) {
+ this.success = false;
+ String stacktrace = Throwables.getStackTraceAsString(thrown);
+ addDiagnosticInfo("exception", thrown.toString());
+ addDiagnosticInfo("stacktrace", stacktrace);
+ }
+
+ public void addDiagnosticInfo(String key, String value) {
+ diagnostics.put(key, value);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("name", name)
+ .add("success", success)
+ .add("timestamp", timestamp)
+ .add("date", date)
+ .add("hostname", hostname)
+ .add("committer", committer)
+ .add("description", description)
+ .add("jobId", jobId)
+ .add("filenames", StringUtils.join(",", filenames))
+ .toString();
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ @Override
+ public byte[] serialize() throws IOException {
+ return CODEC.toBytes(this);
+ }
+
+ public static SuccessData deserialize(byte[] data) throws IOException {
+ return CODEC.fromBytes(data);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, success, timestamp, date, hostname, committer, description, jobId, filenames);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ } else if (!(o instanceof SuccessData)) {
+ return false;
+ }
+ SuccessData that = (SuccessData) o;
+ return Objects.equals(name, that.name)
+ && Objects.equals(success, that.success)
+ && Objects.equals(timestamp, that.timestamp)
+ && Objects.equals(date, that.date)
+ && Objects.equals(hostname, that.hostname)
+ && Objects.equals(committer, that.committer)
+ && Objects.equals(description, that.description)
+ && Objects.equals(jobId, that.jobId)
+ && Objects.equals(filenames, that.filenames);
+ }
+
+ public static class Builder {
+ private String name = SuccessData.class.getName();
+ private boolean success = true;
+ private long timestamp;
+ private String date;
+ private String hostname;
+ private String committer;
+ private String description;
+ private String jobId;
+ private final List<String> filenames = Lists.newArrayList();
+
+ public Builder setName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder setSuccess(boolean success) {
+ this.success = success;
+ return this;
+ }
+
+ public Builder setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ return this;
+ }
+
+ public Builder setDate(String date) {
+ this.date = date;
+ return this;
+ }
+
+ public Builder setHostname(String hostname) {
+ this.hostname = hostname;
+ return this;
+ }
+
+ public Builder setCommitter(String committer) {
+ this.committer = committer;
+ return this;
+ }
+
+ public Builder setDescription(String description) {
+ this.description = description;
+ return this;
+ }
+
+ public Builder setJobId(String jobId) {
+ this.jobId = jobId;
+ return this;
+ }
+
+ public Builder addFileNames(Iterable<String> newFileNames) {
+ if (newFileNames != null) {
+ Iterables.addAll(this.filenames, newFileNames);
+ }
+ return this;
+ }
+
+ public SuccessData build() {
+ return new SuccessData(
+ name, success, timestamp,
+ date, hostname, committer,
+ description, jobId, filenames);
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/mapred/Committer.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/mapred/Committer.java
new file mode 100644
index 00000000000..5a512d71acc
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/mapred/Committer.java
@@ -0,0 +1,184 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit.mapred;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.commit.CommitUtils;
+import org.apache.hadoop.mapred.FileOutputCommitter;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.TaskAttemptContext;
+import org.apache.hadoop.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class Committer extends FileOutputCommitter {
+ private static final Logger LOG = LoggerFactory.getLogger(Committer.class);
+ private org.apache.hadoop.mapreduce.OutputCommitter wrapped = null;
+
+ private static Path getOutputPath(JobContext context) {
+ JobConf conf = context.getJobConf();
+ return FileOutputFormat.getOutputPath(conf);
+ }
+
+ private static Path getOutputPath(TaskAttemptContext context) {
+ JobConf conf = context.getJobConf();
+ return FileOutputFormat.getOutputPath(conf);
+ }
+
+ private org.apache.hadoop.mapreduce.OutputCommitter getWrapped(JobContext context) throws IOException {
+ if(wrapped == null) {
+ wrapped = CommitUtils.supportObjectStorageCommit(context.getConfiguration(), getOutputPath(context))
+ ? new org.apache.hadoop.fs.tosfs.commit.Committer(getOutputPath(context), context)
+ : new org.apache.hadoop.mapred.FileOutputCommitter();
+ LOG.debug("Using OutputCommitter implementation {}", wrapped.getClass().getName());
+ }
+ return wrapped;
+ }
+
+ @InterfaceAudience.Private
+ @Override
+ public Path getTaskAttemptPath(TaskAttemptContext context) throws IOException {
+ Path out = getOutputPath(context);
+ return out == null ? null : getTaskAttemptPath(context, out);
+ }
+
+ private org.apache.hadoop.mapreduce.OutputCommitter getWrapped(TaskAttemptContext context) throws IOException {
+ if(wrapped == null) {
+ wrapped = CommitUtils.supportObjectStorageCommit(context.getConfiguration(), getOutputPath(context))
+ ? new org.apache.hadoop.fs.tosfs.commit.Committer(getOutputPath(context), context)
+ : new org.apache.hadoop.mapred.FileOutputCommitter();
+ }
+ return wrapped;
+ }
+
+ @Override
+ public Path getWorkPath(TaskAttemptContext context, Path outputPath)
+ throws IOException {
+ if (getWrapped(context) instanceof org.apache.hadoop.fs.tosfs.commit.Committer) {
+ return ((org.apache.hadoop.fs.tosfs.commit.Committer) getWrapped(context)).getWorkPath();
+ }
+ return super.getWorkPath(context, outputPath);
+ }
+
+ private Path getTaskAttemptPath(TaskAttemptContext context, Path out) throws IOException {
+ Path workPath = FileOutputFormat.getWorkOutputPath(context.getJobConf());
+ if(workPath == null && out != null) {
+ if (getWrapped(context) instanceof org.apache.hadoop.fs.tosfs.commit.Committer) {
+ return CommitUtils.magicTaskAttemptPath(context, getOutputPath(context));
+ } else {
+ return org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
+ .getTaskAttemptPath(context, out);
+ }
+ }
+ return workPath;
+ }
+
+ @Override
+ public void setupJob(JobContext context) throws IOException {
+ getWrapped(context).setupJob(context);
+ }
+
+ @Override
+ public void commitJob(JobContext context) throws IOException {
+ getWrapped(context).commitJob(context);
+ }
+
+ @Override
+ @Deprecated
+ public void cleanupJob(JobContext context) throws IOException {
+ getWrapped(context).cleanupJob(context);
+ }
+
+ @Override
+ public void abortJob(JobContext context, int runState)
+ throws IOException {
+ JobStatus.State state;
+ if(runState == JobStatus.State.RUNNING.getValue()) {
+ state = JobStatus.State.RUNNING;
+ } else if(runState == JobStatus.State.SUCCEEDED.getValue()) {
+ state = JobStatus.State.SUCCEEDED;
+ } else if(runState == JobStatus.State.FAILED.getValue()) {
+ state = JobStatus.State.FAILED;
+ } else if(runState == JobStatus.State.PREP.getValue()) {
+ state = JobStatus.State.PREP;
+ } else if(runState == JobStatus.State.KILLED.getValue()) {
+ state = JobStatus.State.KILLED;
+ } else {
+ throw new IllegalArgumentException(runState+" is not a valid runState.");
+ }
+ getWrapped(context).abortJob(context, state);
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext context) throws IOException {
+ getWrapped(context).setupTask(context);
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext context) throws IOException {
+ getWrapped(context).commitTask(context);
+ }
+
+ @Override
+ public void abortTask(TaskAttemptContext context) throws IOException {
+ getWrapped(context).abortTask(context);
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext context)
+ throws IOException {
+ return getWrapped(context).needsTaskCommit(context);
+ }
+
+ @Override
+ @Deprecated
+ public boolean isRecoverySupported() {
+ return false;
+ }
+
+ @Override
+ public boolean isCommitJobRepeatable(JobContext context) throws IOException {
+ return getWrapped(context).isCommitJobRepeatable(context);
+ }
+
+ @Override
+ public boolean isRecoverySupported(JobContext context) throws IOException {
+ return getWrapped(context).isRecoverySupported(context);
+ }
+
+ @Override
+ public void recoverTask(TaskAttemptContext context)
+ throws IOException {
+ getWrapped(context).recoverTask(context);
+ }
+
+ public String jobId() {
+ Preconditions.checkNotNull(wrapped, "Encountered uninitialized job committer.");
+ return wrapped instanceof org.apache.hadoop.fs.tosfs.commit.Committer ? ((org.apache.hadoop.fs.tosfs.commit.Committer) wrapped).jobId() : null;
+ }
+
+ public Path getWorkPath() {
+ Preconditions.checkNotNull(wrapped, "Encountered uninitialized job committer.");
+ return wrapped instanceof org.apache.hadoop.fs.tosfs.commit.Committer ? ((org.apache.hadoop.fs.tosfs.commit.Committer) wrapped).getWorkPath() : null;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/ops/PendingOps.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/ops/PendingOps.java
new file mode 100644
index 00000000000..09dffdeb728
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/ops/PendingOps.java
@@ -0,0 +1,43 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit.ops;
+
+import org.apache.hadoop.fs.tosfs.commit.Pending;
+
+public interface PendingOps {
+ /**
+ * Revert the committed {@link Pending}, usually we need to remove or delete the committed files.
+ *
+ * @param commit to revert.
+ */
+ void revert(Pending commit);
+
+ /**
+ * Abort the uncommitted {@link Pending}, to prevent any further committing.
+ *
+ * @param commit to abort.
+ */
+ void abort(Pending commit);
+
+ /**
+ * Commit the {@link Pending} files to be visible. If we want to revert this completed result, please just use
+ * {@link PendingOps#revert(Pending)} to revert this commit.
+ *
+ * @param commit to be visible.
+ */
+ void commit(Pending commit);
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/ops/PendingOpsFactory.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/ops/PendingOpsFactory.java
new file mode 100644
index 00000000000..e84cb62837c
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/ops/PendingOpsFactory.java
@@ -0,0 +1,40 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit.ops;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+
+public class PendingOpsFactory {
+ public static final String PENDING_OPS_IMPL = "pending.ops.impl";
+ public static final String DEFAULT_PENDING_OPS_IMPL = RawPendingOps.class.getName();
+
+ private PendingOpsFactory() {
+ }
+
+ public static PendingOps create(FileSystem fs, ObjectStorage storage) {
+ try {
+ String opsImpl = fs.getConf().get(PENDING_OPS_IMPL, DEFAULT_PENDING_OPS_IMPL);
+ Class<?> clazz = Class.forName(opsImpl);
+ return (PendingOps) clazz
+ .getDeclaredConstructor(FileSystem.class, ObjectStorage.class)
+ .newInstance(fs, storage);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/ops/RawPendingOps.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/ops/RawPendingOps.java
new file mode 100644
index 00000000000..74fec4ec0a5
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/commit/ops/RawPendingOps.java
@@ -0,0 +1,54 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit.ops;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.tosfs.commit.Pending;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * PendingOps will revert, abort or commit the given {@link Pending} commit.
+ */
+public class RawPendingOps implements PendingOps {
+ private static final Logger LOG = LoggerFactory.getLogger(RawPendingOps.class);
+
+ private final ObjectStorage storage;
+
+ /**
+ * Constructor for {@link PendingOpsFactory} to reflect a new instance.
+ */
+ public RawPendingOps(FileSystem fs, ObjectStorage storage) {
+ this.storage = storage;
+ }
+
+ public void revert(Pending commit) {
+ LOG.info("Revert the commit by deleting the object key - {}", commit);
+ storage.delete(commit.destKey());
+ }
+
+ public void abort(Pending commit) {
+ LOG.info("Abort the commit by aborting multipart upload - {}", commit);
+ storage.abortMultipartUpload(commit.destKey(), commit.uploadId());
+ }
+
+ public void commit(Pending commit) {
+ LOG.info("Commit by completing the multipart uploads - {}", commit);
+ storage.completeUpload(commit.destKey(), commit.uploadId(), commit.parts());
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/common/Bytes.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/common/Bytes.java
new file mode 100644
index 00000000000..eb632050a53
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/common/Bytes.java
@@ -0,0 +1,182 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.common;
+
+import org.apache.hadoop.util.Preconditions;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+
+public class Bytes {
+ private Bytes() {
+ }
+
+ public static final byte[] EMPTY_BYTES = new byte[0];
+
+ // Encode basic Java types into big-endian binaries.
+
+ public static byte[] toBytes(boolean b) {
+ return new byte[] { b ? (byte) -1 : (byte) 0 };
+ }
+
+ public static byte[] toBytes(byte b) {
+ return new byte[] { b };
+ }
+
+ public static byte[] toBytes(short val) {
+ byte[] b = new byte[2];
+ for (int i = 1; i >= 0; i--) {
+ b[i] = (byte) val;
+ val >>>= 8;
+ }
+ return b;
+ }
+
+ public static byte[] toBytes(int val) {
+ byte[] b = new byte[4];
+ for (int i = 3; i >= 0; i--) {
+ b[i] = (byte) val;
+ val >>>= 8;
+ }
+ return b;
+ }
+
+ public static byte[] toBytes(long val) {
+ byte[] b = new byte[8];
+ for (int i = 7; i >= 0; i--) {
+ b[i] = (byte) val;
+ val >>>= 8;
+ }
+ return b;
+ }
+
+ public static byte[] toBytes(String s) {
+ return s.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // Decode big-endian binaries into basic Java types.
+
+ public static boolean toBoolean(byte[] b) {
+ return toBoolean(b, 0, 1);
+ }
+
+ public static boolean toBoolean(byte[] b, int off) {
+ return toBoolean(b, off, 1);
+ }
+
+ public static boolean toBoolean(byte[] b, int off, int len) {
+ Preconditions.checkArgument(len == 1, "Invalid len: %s", len);
+ return b[off] != (byte) 0;
+ }
+
+ public static byte toByte(byte[] b) {
+ return b[0];
+ }
+
+ public static byte toByte(byte[] b, int off) {
+ return b[off];
+ }
+
+ public static short toShort(byte[] b) {
+ return toShort(b, 0, 2);
+ }
+
+ public static short toShort(byte[] b, int off) {
+ return toShort(b, off, 2);
+ }
+
+ public static short toShort(byte[] b, int off, int len) {
+ Preconditions.checkArgument(len == 2, "Invalid len: %s", len);
+ Preconditions.checkArgument(off >= 0 && off + len <= b.length,
+ "Invalid off: %s, len: %s, array size: %s", off, len, b.length);
+ short n = 0;
+ n = (short) ((n ^ b[off]) & 0xFF);
+ n = (short) (n << 8);
+ n ^= (short) (b[off + 1] & 0xFF);
+ return n;
+ }
+
+ public static int toInt(byte[] b) {
+ return toInt(b, 0, 4);
+ }
+
+ public static int toInt(byte[] b, int off) {
+ return toInt(b, off, 4);
+ }
+
+ public static int toInt(byte[] b, int off, int len) {
+ Preconditions.checkArgument(len == 4, "Invalid len: %s", len);
+ Preconditions.checkArgument(off >= 0 && off + len <= b.length,
+ "Invalid off: %s, len: %s, array size: %s", off, len, b.length);
+ int n = 0;
+ for (int i = off; i < (off + len); i++) {
+ n <<= 8;
+ n ^= b[i] & 0xFF;
+ }
+ return n;
+ }
+
+ public static int toInt(ByteBuffer b) {
+ Preconditions.checkArgument(4 <= b.remaining(),
+ "Invalid ByteBuffer which remaining must be >= 4, but is: %s", b.remaining());
+ int n = 0;
+ int off = b.position();
+ for (int i = off; i < (off + 4); i++) {
+ n <<= 8;
+ n ^= b.get(i) & 0xFF;
+ }
+ return n;
+ }
+
+ public static long toLong(byte[] b) {
+ return toLong(b, 0, 8);
+ }
+
+ public static long toLong(byte[] b, int off) {
+ return toLong(b, off, 8);
+ }
+
+ public static long toLong(byte[] b, int off, int len) {
+ Preconditions.checkArgument(len == 8, "Invalid len: %s", len);
+ Preconditions.checkArgument(off >= 0 && off + len <= b.length,
+ "Invalid off: %s, len: %s, array size: %s", off, len, b.length);
+ long l = 0;
+ for (int i = off; i < off + len; i++) {
+ l <<= 8;
+ l ^= b[i] & 0xFF;
+ }
+ return l;
+ }
+
+ public static byte[] toBytes(byte[] b, int off, int len) {
+ Preconditions.checkArgument(off >= 0, "off %s must be >=0", off);
+ Preconditions.checkArgument(len >= 0, "len %s must be >= 0", len);
+ Preconditions.checkArgument(off + len <= b.length, "off (%s) + len (%s) must be <= %s", off,
+ len, b.length);
+ byte[] data = new byte[len];
+ System.arraycopy(b, off, data, 0, len);
+ return data;
+ }
+
+ public static String toString(byte[] b) {
+ return new String(b, StandardCharsets.UTF_8);
+ }
+
+ public static String toString(byte[] b, int off, int len) {
+ return new String(b, off, len, StandardCharsets.UTF_8);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/common/Chain.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/common/Chain.java
new file mode 100644
index 00000000000..b5465f674bd
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/common/Chain.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.common;
+
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Queues;
+import org.apache.hadoop.util.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Deque;
+import java.util.List;
+import java.util.Queue;
+import java.util.function.Predicate;
+
+public class Chain<T extends Closeable> implements Closeable {
+ private static final Logger LOG = LoggerFactory.getLogger(Chain.class);
+
+ private final List<IOException> suppressed = Lists.newArrayList();
+ private final Queue<ItemFactory<T>> queue;
+ private final Predicate<IOException> shouldContinue;
+ private T curItem;
+
+ private Chain(Deque<ItemFactory<T>> queue, Predicate<IOException> shouldContinue) {
+ this.queue = queue;
+ this.shouldContinue = shouldContinue;
+ this.curItem = null;
+ }
+
+ public <R> R run(Task<T, R> task) throws IOException {
+ while (true) {
+ if (curItem == null && !nextItem()) {
+ IOException ex = new IOException("Failed to run task after attempt all items");
+ suppressed.forEach(ex::addSuppressed);
+ throw ex;
+ }
+
+ try {
+ return task.run(curItem);
+ } catch (IOException e) {
+ LOG.debug("Encounter exception while running task with item {}", curItem, e);
+ // Resetting the current caller to be null, for triggering the next round election.
+ if (curItem != null) {
+ CommonUtils.runQuietly(curItem::close);
+ curItem = null;
+ }
+ suppressed.add(e);
+
+ if (shouldContinue != null && !shouldContinue.test(e)) {
+ IOException ex = new IOException("Failed to run the chain since the encountered error not retryable.");
+ suppressed.forEach(ex::addSuppressed);
+ throw ex;
+ }
+ }
+ }
+ }
+
+ public T curItem() {
+ return curItem;
+ }
+
+ private boolean nextItem() {
+ if (curItem != null) {
+ CommonUtils.runQuietly(curItem::close);
+ curItem = null;
+ }
+
+ while (!queue.isEmpty()) {
+ ItemFactory<T> nextFactory = queue.poll();
+ try {
+ curItem = nextFactory.newItem();
+ return true;
+ } catch (IOException e) {
+ curItem = null;
+ LOG.debug("Failed to create new item", e);
+ suppressed.add(e);
+ }
+ }
+
+ return false;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (curItem != null) {
+ curItem.close();
+ }
+ }
+
+ public interface ItemFactory<T extends Closeable> {
+ T newItem() throws IOException;
+ }
+
+ public interface Task<T extends Closeable, R> {
+ R run(T call) throws IOException;
+ }
+
+ public static class Builder<T extends Closeable> {
+ private final Deque<ItemFactory<T>> factories = Queues.newArrayDeque();
+ private Predicate<IOException> shouldContinue;
+
+ public Builder<T> addFirst(ItemFactory<T> factory) {
+ factories.addFirst(factory);
+ return this;
+ }
+
+ public Builder<T> addLast(ItemFactory<T> factory) {
+ factories.addLast(factory);
+ return this;
+ }
+
+ public Builder<T> shouldContinue(Predicate<IOException> shouldContinue) {
+ this.shouldContinue = shouldContinue;
+ return this;
+ }
+
+ public Chain<T> build() throws IOException {
+ Chain<T> chain = new Chain<>(factories, shouldContinue);
+
+ // Do nothing in the chain task to initialize the first item.
+ chain.run(item -> null);
+ return chain;
+ }
+ }
+
+ public static <T extends Closeable> Builder<T> builder() {
+ return new Builder<>();
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/common/Tasks.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/common/Tasks.java
new file mode 100644
index 00000000000..563d7813cd3
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/common/Tasks.java
@@ -0,0 +1,590 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.common;
+
+import org.apache.hadoop.util.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+/**
+ * Copied from Apache Iceberg, please see:
+ * <a href="https://github.com/apache/iceberg/blob/master/core/src/main/java/org/apache/iceberg/util/Tasks.java">
+ * Tasks</a>
+ */
+public class Tasks {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Tasks.class);
+
+ private Tasks() {
+ }
+
+ public static class UnrecoverableException extends RuntimeException {
+ public UnrecoverableException(String message) {
+ super(message);
+ }
+
+ public UnrecoverableException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public UnrecoverableException(Throwable cause) {
+ super(cause);
+ }
+ }
+
+ public interface FailureTask<I, E extends Exception> {
+ void run(I item, Exception exception) throws E;
+ }
+
+ public interface Task<I, E extends Exception> {
+ void run(I item) throws E;
+ }
+
+ public static class Builder<I> {
+ private final Iterable<I> items;
+ private ExecutorService service = null;
+ private FailureTask<I, ?> onFailure = null;
+ private boolean stopOnFailure = false;
+ private boolean throwFailureWhenFinished = true;
+ private Task<I, ?> revertTask = null;
+ private boolean stopRevertsOnFailure = false;
+ private Task<I, ?> abortTask = null;
+ private boolean stopAbortsOnFailure = false;
+
+ // retry settings
+ private List<Class<? extends Exception>> stopRetryExceptions =
+ Lists.newArrayList(UnrecoverableException.class);
+ private List<Class<? extends Exception>> onlyRetryExceptions = null;
+ private Predicate<Exception> shouldRetryPredicate = null;
+ private int maxAttempts = 1; // not all operations can be retried
+ private long minSleepTimeMs = 1000; // 1 second
+ private long maxSleepTimeMs = 600000; // 10 minutes
+ private long maxDurationMs = 600000; // 10 minutes
+ private double scaleFactor = 2.0; // exponential
+
+ public Builder(Iterable<I> items) {
+ this.items = items;
+ }
+
+ public Builder<I> executeWith(ExecutorService svc) {
+ this.service = svc;
+ return this;
+ }
+
+ public Builder<I> onFailure(FailureTask<I, ?> task) {
+ this.onFailure = task;
+ return this;
+ }
+
+ public Builder<I> stopOnFailure() {
+ this.stopOnFailure = true;
+ return this;
+ }
+
+ public Builder<I> throwFailureWhenFinished() {
+ this.throwFailureWhenFinished = true;
+ return this;
+ }
+
+ public Builder<I> throwFailureWhenFinished(boolean throwWhenFinished) {
+ this.throwFailureWhenFinished = throwWhenFinished;
+ return this;
+ }
+
+ public Builder<I> suppressFailureWhenFinished() {
+ this.throwFailureWhenFinished = false;
+ return this;
+ }
+
+ public Builder<I> revertWith(Task<I, ?> task) {
+ this.revertTask = task;
+ return this;
+ }
+
+ public Builder<I> stopRevertsOnFailure() {
+ this.stopRevertsOnFailure = true;
+ return this;
+ }
+
+ public Builder<I> abortWith(Task<I, ?> task) {
+ this.abortTask = task;
+ return this;
+ }
+
+ public Builder<I> stopAbortsOnFailure() {
+ this.stopAbortsOnFailure = true;
+ return this;
+ }
+
+ @SafeVarargs public final Builder<I> stopRetryOn(Class<? extends Exception>... exceptions) {
+ stopRetryExceptions.addAll(Arrays.asList(exceptions));
+ return this;
+ }
+
+ public Builder<I> shouldRetryTest(Predicate<Exception> shouldRetry) {
+ this.shouldRetryPredicate = shouldRetry;
+ return this;
+ }
+
+ public Builder<I> noRetry() {
+ this.maxAttempts = 1;
+ return this;
+ }
+
+ public Builder<I> retry(int nTimes) {
+ this.maxAttempts = nTimes + 1;
+ return this;
+ }
+
+ public Builder<I> onlyRetryOn(Class<? extends Exception> exception) {
+ this.onlyRetryExceptions = Collections.singletonList(exception);
+ return this;
+ }
+
+ @SafeVarargs public final Builder<I> onlyRetryOn(Class<? extends Exception>... exceptions) {
+ this.onlyRetryExceptions = Lists.newArrayList(exceptions);
+ return this;
+ }
+
+ public Builder<I> exponentialBackoff(long backoffMinSleepTimeMs, long backoffMaxSleepTimeMs,
+ long backoffMaxRetryTimeMs, double backoffScaleFactor) {
+ this.minSleepTimeMs = backoffMinSleepTimeMs;
+ this.maxSleepTimeMs = backoffMaxSleepTimeMs;
+ this.maxDurationMs = backoffMaxRetryTimeMs;
+ this.scaleFactor = backoffScaleFactor;
+ return this;
+ }
+
+ public boolean run(Task<I, RuntimeException> task) {
+ return run(task, RuntimeException.class);
+ }
+
+ public <E extends Exception> boolean run(Task<I, E> task, Class<E> exceptionClass) throws E {
+ if (service != null) {
+ return runParallel(task, exceptionClass);
+ } else {
+ return runSingleThreaded(task, exceptionClass);
+ }
+ }
+
+ @SuppressWarnings("checkstyle:CyclomaticComplexity") private <E extends Exception> boolean runSingleThreaded(
+ Task<I, E> task, Class<E> exceptionClass) throws E {
+ List<I> succeeded = Lists.newArrayList();
+ List<Throwable> exceptions = Lists.newArrayList();
+
+ Iterator<I> iterator = items.iterator();
+ boolean threw = true;
+ try {
+ while (iterator.hasNext()) {
+ I item = iterator.next();
+ try {
+ runTaskWithRetry(task, item);
+ succeeded.add(item);
+ } catch (Exception e) {
+ exceptions.add(e);
+
+ if (onFailure != null) {
+ tryRunOnFailure(item, e);
+ }
+
+ if (stopOnFailure) {
+ break;
+ }
+ }
+ }
+
+ threw = false;
+
+ } finally {
+ // threw handles exceptions that were *not* caught by the catch block,
+ // and exceptions that were caught and possibly handled by onFailure
+ // are kept in exceptions.
+ if (threw || !exceptions.isEmpty()) {
+ if (revertTask != null) {
+ boolean failed = false;
+ for (I item : succeeded) {
+ try {
+ revertTask.run(item);
+ } catch (Exception e) {
+ failed = true;
+ LOG.error("Failed to revert task", e);
+ // keep going
+ }
+ if (stopRevertsOnFailure && failed) {
+ break;
+ }
+ }
+ }
+
+ if (abortTask != null) {
+ boolean failed = false;
+ while (iterator.hasNext()) {
+ try {
+ abortTask.run(iterator.next());
+ } catch (Exception e) {
+ failed = true;
+ LOG.error("Failed to abort task", e);
+ // keep going
+ }
+ if (stopAbortsOnFailure && failed) {
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ if (throwFailureWhenFinished && !exceptions.isEmpty()) {
+ Tasks.throwOne(exceptions, exceptionClass);
+ } else if (throwFailureWhenFinished && threw) {
+ throw new RuntimeException("Task set failed with an uncaught throwable");
+ }
+
+ return !threw;
+ }
+
+ private void tryRunOnFailure(I item, Exception failure) {
+ try {
+ onFailure.run(item, failure);
+ } catch (Exception failException) {
+ failure.addSuppressed(failException);
+ LOG.error("Failed to clean up on failure", failException);
+ // keep going
+ }
+ }
+
+ @SuppressWarnings("checkstyle:CyclomaticComplexity") private <E extends Exception> boolean runParallel(
+ final Task<I, E> task, Class<E> exceptionClass) throws E {
+ final Queue<I> succeeded = new ConcurrentLinkedQueue<>();
+ final Queue<Throwable> exceptions = new ConcurrentLinkedQueue<>();
+ final AtomicBoolean taskFailed = new AtomicBoolean(false);
+ final AtomicBoolean abortFailed = new AtomicBoolean(false);
+ final AtomicBoolean revertFailed = new AtomicBoolean(false);
+
+ List<Future<?>> futures = Lists.newArrayList();
+
+ for (final I item : items) {
+ // submit a task for each item that will either run or abort the task
+ futures.add(service.submit(() -> {
+ if (!(stopOnFailure && taskFailed.get())) {
+ // run the task with retries
+ boolean threw = true;
+ try {
+ runTaskWithRetry(task, item);
+
+ succeeded.add(item);
+
+ threw = false;
+
+ } catch (Exception e) {
+ taskFailed.set(true);
+ exceptions.add(e);
+
+ if (onFailure != null) {
+ tryRunOnFailure(item, e);
+ }
+ } finally {
+ if (threw) {
+ taskFailed.set(true);
+ }
+ }
+
+ } else if (abortTask != null) {
+ // abort the task instead of running it
+ if (stopAbortsOnFailure && abortFailed.get()) {
+ return;
+ }
+
+ boolean failed = true;
+ try {
+ abortTask.run(item);
+ failed = false;
+ } catch (Exception e) {
+ LOG.error("Failed to abort task", e);
+ // swallow the exception
+ } finally {
+ if (failed) {
+ abortFailed.set(true);
+ }
+ }
+ }
+ }));
+ }
+
+ // let the above tasks complete (or abort)
+ exceptions.addAll(waitFor(futures));
+ futures.clear();
+
+ if (taskFailed.get() && revertTask != null) {
+ // at least one task failed, revert any that succeeded
+ for (final I item : succeeded) {
+ futures.add(service.submit(() -> {
+ if (stopRevertsOnFailure && revertFailed.get()) {
+ return;
+ }
+
+ boolean failed = true;
+ try {
+ revertTask.run(item);
+ failed = false;
+ } catch (Exception e) {
+ LOG.error("Failed to revert task", e);
+ // swallow the exception
+ } finally {
+ if (failed) {
+ revertFailed.set(true);
+ }
+ }
+ }));
+ }
+
+ // let the revert tasks complete
+ exceptions.addAll(waitFor(futures));
+ }
+
+ if (throwFailureWhenFinished && !exceptions.isEmpty()) {
+ Tasks.throwOne(exceptions, exceptionClass);
+ } else if (throwFailureWhenFinished && taskFailed.get()) {
+ throw new RuntimeException("Task set failed with an uncaught throwable");
+ }
+
+ return !taskFailed.get();
+ }
+
+ @SuppressWarnings("checkstyle:CyclomaticComplexity") private <E extends Exception> void runTaskWithRetry(
+ Task<I, E> task, I item) throws E {
+ long start = System.currentTimeMillis();
+ int attempt = 0;
+ while (true) {
+ attempt += 1;
+ try {
+ task.run(item);
+ break;
+
+ } catch (Exception e) {
+ long durationMs = System.currentTimeMillis() - start;
+ if (attempt >= maxAttempts || (durationMs > maxDurationMs && attempt > 1)) {
+ if (durationMs > maxDurationMs) {
+ LOG.info("Stopping retries after {} ms", durationMs);
+ }
+ throw e;
+ }
+
+ if (shouldRetryPredicate != null) {
+ if (!shouldRetryPredicate.test(e)) {
+ throw e;
+ }
+
+ } else if (onlyRetryExceptions != null) {
+ // if onlyRetryExceptions are present, then this retries if one is found
+ boolean matchedRetryException = false;
+ for (Class<? extends Exception> exClass : onlyRetryExceptions) {
+ if (exClass.isInstance(e)) {
+ matchedRetryException = true;
+ break;
+ }
+ }
+ if (!matchedRetryException) {
+ throw e;
+ }
+
+ } else {
+ // otherwise, always retry unless one of the stop exceptions is found
+ for (Class<? extends Exception> exClass : stopRetryExceptions) {
+ if (exClass.isInstance(e)) {
+ throw e;
+ }
+ }
+ }
+
+ int delayMs =
+ (int) Math.min(minSleepTimeMs * Math.pow(scaleFactor, attempt - 1), maxSleepTimeMs);
+ int jitter = ThreadLocalRandom.current().nextInt(Math.max(1, (int) (delayMs * 0.1)));
+
+ LOG.warn("Retrying task after failure: {}", e.getMessage(), e);
+
+ try {
+ TimeUnit.MILLISECONDS.sleep(delayMs + jitter);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(ie);
+ }
+ }
+ }
+ }
+ }
+
+ @SuppressWarnings("checkstyle:CyclomaticComplexity") private static Collection<Throwable> waitFor(
+ Collection<Future<?>> futures) {
+ while (true) {
+ int numFinished = 0;
+ for (Future<?> future : futures) {
+ if (future.isDone()) {
+ numFinished += 1;
+ }
+ }
+
+ if (numFinished == futures.size()) {
+ List<Throwable> uncaught = Lists.newArrayList();
+ // all of the futures are done, get any uncaught exceptions
+ for (Future<?> future : futures) {
+ try {
+ future.get();
+
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while getting future results", e);
+ for (Throwable t : uncaught) {
+ e.addSuppressed(t);
+ }
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+
+ } catch (CancellationException e) {
+ // ignore cancellations
+
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ if (Error.class.isInstance(cause)) {
+ for (Throwable t : uncaught) {
+ cause.addSuppressed(t);
+ }
+ throw (Error) cause;
+ }
+
+ if (cause != null) {
+ uncaught.add(e);
+ }
+
+ LOG.warn("Task threw uncaught exception", cause);
+ }
+ }
+
+ return uncaught;
+
+ } else {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for tasks to finish", e);
+
+ for (Future<?> future : futures) {
+ future.cancel(true);
+ }
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
+ /**
+ * A range, [ 0, size )
+ */
+ private static class Range implements Iterable<Integer> {
+ private final int size;
+
+ Range(int size) {
+ this.size = size;
+ }
+
+ @Override
+ public Iterator<Integer> iterator() {
+ return new Iterator<Integer>() {
+ private int current = 0;
+
+ @Override
+ public boolean hasNext() {
+ return current < size;
+ }
+
+ @Override
+ public Integer next() {
+ int ret = current;
+ current += 1;
+ return ret;
+ }
+ };
+ }
+ }
+
+ public static Builder<Integer> range(int upTo) {
+ return new Builder<>(new Range(upTo));
+ }
+
+ public static <I> Builder<I> foreach(Iterator<I> items) {
+ return new Builder<>(() -> items);
+ }
+
+ public static <I> Builder<I> foreach(Iterable<I> items) {
+ return new Builder<>(items);
+ }
+
+ @SafeVarargs public static <I> Builder<I> foreach(I... items) {
+ return new Builder<>(Arrays.asList(items));
+ }
+
+ @SuppressWarnings("StreamToIterable") public static <I> Builder<I> foreach(Stream<I> items) {
+ return new Builder<>(items::iterator);
+ }
+
+ private static <E extends Exception> void throwOne(Collection<Throwable> exceptions,
+ Class<E> allowedException) throws E {
+ Iterator<Throwable> iter = exceptions.iterator();
+ Throwable exception = iter.next();
+ Class<? extends Throwable> exceptionClass = exception.getClass();
+
+ while (iter.hasNext()) {
+ Throwable other = iter.next();
+ if (!exceptionClass.isInstance(other)) {
+ exception.addSuppressed(other);
+ }
+ }
+
+ castAndThrow(exception, allowedException);
+ }
+
+ @SuppressWarnings("unchecked") public static <E extends Exception> void castAndThrow(
+ Throwable exception, Class<E> exceptionClass) throws E {
+ if (exception instanceof RuntimeException) {
+ throw (RuntimeException) exception;
+ } else if (exception instanceof Error) {
+ throw (Error) exception;
+ } else if (exceptionClass.isInstance(exception)) {
+ throw (E) exception;
+ }
+ throw new RuntimeException(exception);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/common/ThreadPools.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/common/ThreadPools.java
new file mode 100644
index 00000000000..df3bc81cd8e
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/common/ThreadPools.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.common;
+
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Copied from Apache Iceberg, please see:
+ * <a href="https://github.com/apache/iceberg/blob/master/core/src/main/java/org/apache/iceberg/util/ThreadPools.java">
+ * ThreadPools</a>
+ */
+public class ThreadPools {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ThreadPools.class);
+
+ private ThreadPools() {
+ }
+
+ public static final String WORKER_THREAD_POOL_SIZE_PROP = "tos.worker.num-threads";
+
+ public static final int WORKER_THREAD_POOL_SIZE =
+ poolSize(Math.max(2, Runtime.getRuntime().availableProcessors()));
+
+ private static final ExecutorService WORKER_POOL = newWorkerPool("tos-default-worker-pool");
+
+ public static ExecutorService defaultWorkerPool() {
+ return WORKER_POOL;
+ }
+
+ public static ExecutorService newWorkerPool(String namePrefix) {
+ return newWorkerPool(namePrefix, WORKER_THREAD_POOL_SIZE);
+ }
+
+ public static ExecutorService newWorkerPool(String namePrefix, int poolSize) {
+ return Executors.newFixedThreadPool(poolSize, newDaemonThreadFactory(namePrefix));
+ }
+
+ public static ScheduledExecutorService newScheduleWorkerPool(String namePrefix, int poolSize) {
+ return Executors.newScheduledThreadPool(poolSize, newDaemonThreadFactory(namePrefix));
+ }
+
+ /**
+ * Helper routine to shutdown a {@link ExecutorService}. Will wait up to a
+ * certain timeout for the ExecutorService to gracefully shutdown. If the
+ * ExecutorService did not shutdown and there are still tasks unfinished after
+ * the timeout period, the ExecutorService will be notified to forcibly shut
+ * down. Another timeout period will be waited before giving up. So, at most,
+ * a shutdown will be allowed to wait up to twice the timeout value before
+ * giving up.
+ * <p>
+ * This method is copied from
+ * {@link org.apache.hadoop.util.concurrent.HadoopExecutors#shutdown(ExecutorService, Logger, long, TimeUnit)}.
+ *
+ * @param executorService ExecutorService to shutdown
+ * @param timeout the maximum time to wait
+ * @param unit the time unit of the timeout argument
+ */
+ public static void shutdown(ExecutorService executorService, long timeout, TimeUnit unit) {
+ if (executorService == null) {
+ return;
+ }
+
+ try {
+ executorService.shutdown();
+ LOG.debug("Gracefully shutting down executor service. Waiting max {} {}", timeout, unit);
+
+ if (!executorService.awaitTermination(timeout, unit)) {
+ LOG.debug(
+ "Executor service has not shutdown yet. Forcing. Will wait up to an additional {} {} for shutdown",
+ timeout, unit);
+ executorService.shutdownNow();
+ }
+
+ if (executorService.awaitTermination(timeout, unit)) {
+ LOG.debug("Succesfully shutdown executor service");
+ } else {
+ LOG.error("Unable to shutdown executor service after timeout {} {}", (2 * timeout), unit);
+ }
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted while attempting to shutdown", e);
+ executorService.shutdownNow();
+ } catch (Exception e) {
+ LOG.warn("Exception closing executor service {}", e.getMessage());
+ LOG.debug("Exception closing executor service", e);
+ throw e;
+ }
+ }
+
+ private static int poolSize(int defaultSize) {
+ String value = System.getProperty(WORKER_THREAD_POOL_SIZE_PROP);
+ if (value != null) {
+ try {
+ return Integer.parseUnsignedInt(value);
+ } catch (NumberFormatException e) {
+ // will return the default
+ }
+ }
+ return defaultSize;
+ }
+
+ public static ThreadFactory newDaemonThreadFactory(String namePrefix) {
+ return new ThreadFactoryBuilder().setDaemon(true).setNameFormat(namePrefix + "-%d")
+ .setUncaughtExceptionHandler(
+ (t, e) -> LOG.error("Thread {} encounter uncaught exception", t, e)).build();
+ }
+
+ public static Thread newDaemonThread(String name, Runnable runnable,
+ UncaughtExceptionHandler handler) {
+ Thread t = new Thread(runnable);
+ t.setName(name);
+ t.setDaemon(true);
+ if (handler != null) {
+ t.setUncaughtExceptionHandler(handler);
+ }
+ return t;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/conf/ArgumentKey.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/conf/ArgumentKey.java
new file mode 100644
index 00000000000..7c7821bcff4
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/conf/ArgumentKey.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.conf;
+
+public class ArgumentKey {
+
+ private final String template;
+
+ public ArgumentKey(String template) {
+ this.template = template;
+ }
+
+ public String key(Object... arguments) {
+ return String.format(template, arguments);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java
new file mode 100644
index 00000000000..990033c361a
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.conf;
+
+import org.apache.hadoop.util.Preconditions;
+
+public class ConfKeys {
+
+ /**
+ * Object storage endpoint to connect to, which should include both region and object domain name.
+ * e.g. 'fs.tos.endpoint'='tos-cn-beijing.volces.com'.
+ */
+ public static final ArgumentKey FS_OBJECT_STORAGE_ENDPOINT = new ArgumentKey("fs.%s.endpoint");
+
+ /**
+ * The region of the object storage, e.g. fs.tos.region. Parsing template "fs.%s.endpoint" to
+ * know the region.
+ */
+ public static final ArgumentKey FS_OBJECT_STORAGE_REGION = new ArgumentKey("fs.%s.region");
+
+ /**
+ * The object storage implementation for the defined scheme. For example, we can delegate the
+ * scheme 'abc' to TOS (or other object storage),and access the TOS object storage as
+ * 'abc://bucket/path/to/key'
+ */
+ public static final ArgumentKey FS_OBJECT_STORAGE_IMPL =
+ new ArgumentKey("fs.objectstorage.%s.impl");
+
+ /**
+ * The batch size of deleting multiple objects per request for the given object storage.
+ * e.g. fs.tos.delete.batch-size
+ */
+ public static final ArgumentKey FS_BATCH_DELETE_SIZE = new ArgumentKey("fs.%s.delete.batch-size");
+ public static final int FS_BATCH_DELETE_SIZE_DEFAULT = 250;
+
+ /**
+ * The multipart upload part size of the given object storage, e.g. fs.tos.multipart.size.
+ */
+ public static final ArgumentKey FS_MULTIPART_SIZE = new ArgumentKey("fs.%s.multipart.size");
+ public static final long FS_MULTIPART_SIZE_DEFAULT = 8L << 20;
+
+ /**
+ * The threshold (larger than this value) to enable multipart upload during copying objects
+ * in the given object storage. If the copied data size is less than threshold, will copy data via
+ * executing copyObject instead of uploadPartCopy. E.g. fs.tos.multipart.copy-threshold
+ */
+ public static final ArgumentKey FS_MULTIPART_COPY_THRESHOLD =
+ new ArgumentKey("fs.%s.multipart.copy-threshold");
+ public static final long FS_MULTIPART_COPY_THRESHOLD_DEFAULT = 5L << 20;
+
+ /**
+ * The threshold which control whether enable multipart upload during writing data to the given
+ * object storage, if the write data size is less than threshold, will write data via simple put
+ * instead of multipart upload. E.g. fs.tos.multipart.threshold.
+ */
+ public static final ArgumentKey FS_MULTIPART_THRESHOLD =
+ new ArgumentKey("fs.%s.multipart.threshold");
+ public static final long FS_MULTIPART_THRESHOLD_DEFAULT = 10 << 20;
+
+ /**
+ * The max byte size which will buffer the staging data in-memory before flushing to the staging
+ * file. It will decrease the random write in local staging disk dramatically if writing plenty of
+ * small files.
+ */
+ public static final ArgumentKey FS_MULTIPART_STAGING_BUFFER_SIZE =
+ new ArgumentKey("fs.%s.multipart.staging-buffer-size");
+ public static final int FS_MULTIPART_STAGING_BUFFER_SIZE_DEFAULT = 4 << 10;
+
+ /**
+ * The multipart upload part staging dir(s) of the given object storage.
+ * e.g. fs.tos.multipart.staging-dir.
+ * Separate the staging dirs with comma if there are many staging dir paths.
+ */
+ public static final ArgumentKey FS_MULTIPART_STAGING_DIR =
+ new ArgumentKey("fs.%s.multipart.staging-dir");
+ public static final String FS_MULTIPART_STAGING_DIR_DEFAULT = defaultDir("multipart-staging-dir");
+
+ /**
+ * True to create the missed parent dir asynchronously during deleting or renaming a file or dir.
+ */
+ public static final ArgumentKey FS_ASYNC_CREATE_MISSED_PARENT =
+ new ArgumentKey("fs.%s.missed.parent.dir.async-create");
+ public static final boolean FS_ASYNC_CREATE_MISSED_PARENT_DEFAULT = true;
+
+ /**
+ * Whether using rename semantic of object storage during rename files, otherwise using
+ * copy + delete.
+ * Please ensure that the object storage support and enable rename semantic and before enable it,
+ * and also ensure grant rename permission to the requester.
+ * If you are using TOS, you have to send putBucketRename request before sending rename request,
+ * otherwise MethodNotAllowed exception will be thrown.
+ */
+ public static final ArgumentKey FS_OBJECT_RENAME_ENABLED =
+ new ArgumentKey("fs.%s.rename.enabled");
+ public static final boolean FS_OBJECT_RENAME_ENABLED_DEFAULT = false;
+
+ /**
+ * The range size when open object storage input stream. Value must be positive.
+ */
+ public static final String FS_OBJECT_STREAM_RANGE_SIZE = "fs.objectstorage.stream.range-size";
+ public static final long FS_OBJECT_STREAM_RANGE_SIZE_DEFAULT = Long.MAX_VALUE;
+
+ /**
+ * The size of thread pool used for running tasks in parallel for the given object fs,
+ * e.g. delete objects, copy files. the key example: fs.tos.task.thread-pool-size.
+ */
+ public static final ArgumentKey FS_TASK_THREAD_POOL_SIZE =
+ new ArgumentKey("fs.%s.task.thread-pool-size");
+ public static final int FS_TASK_THREAD_POOL_SIZE_DEFAULT =
+ Math.max(2, Runtime.getRuntime().availableProcessors());
+
+ /**
+ * The size of thread pool used for uploading multipart in parallel for the given object storage,
+ * e.g. fs.tos.multipart.thread-pool-size
+ */
+ public static final ArgumentKey FS_MULTIPART_THREAD_POOL_SIZE =
+ new ArgumentKey("fs.%s.multipart.thread-pool-size");
+ public static final int FS_MULTIPART_THREAD_POOL_SIZE_DEFAULT =
+ Math.max(2, Runtime.getRuntime().availableProcessors());
+
+ /**
+ * The toggle indicates whether enable checksum during getting file status for the given object.
+ * E.g. fs.tos.checksum.enabled
+ */
+ public static final ArgumentKey FS_CHECKSUM_ENABLED = new ArgumentKey("fs.%s.checksum.enabled");
+ public static final boolean FS_CHECKSUM_ENABLED_DEFAULT = true;
+
+ public static String defaultDir(String basename) {
+ String tmpdir = System.getProperty("java.io.tmpdir");
+ Preconditions.checkNotNull(tmpdir, "System property 'java.io.tmpdir' cannot be null");
+
+ if (tmpdir.endsWith("/")) {
+ return String.format("%s%s", tmpdir, basename);
+ } else {
+ return String.format("%s/%s", tmpdir, basename);
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/conf/FileStoreKeys.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/conf/FileStoreKeys.java
new file mode 100644
index 00000000000..3411821d568
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/conf/FileStoreKeys.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.conf;
+
+import org.apache.hadoop.fs.tosfs.object.ChecksumType;
+
+public class FileStoreKeys {
+
+ /**
+ * The key indicates the name of the filestore checksum algorithm. Specify the algorithm name to
+ * satisfy different storage systems. For example, the hdfs style name is COMPOSITE-CRC32 and
+ * COMPOSITE-CRC32C.
+ */
+ public static final String FS_FILESTORE_CHECKSUM_ALGORITHM = "fs.filestore.checksum-algorithm";
+ public static final String FS_FILESTORE_CHECKSUM_ALGORITHM_DEFAULT = "TOS-CHECKSUM";
+
+ /**
+ * The key indicates how to retrieve file checksum from filestore, error will be thrown if the
+ * configured checksum type is not supported. The supported checksum type is: MD5.
+ */
+ public static final String FS_FILESTORE_CHECKSUM_TYPE = "fs.filestore.checksum-type";
+ public static final String FS_FILESTORE_CHECKSUM_TYPE_DEFAULT = ChecksumType.MD5.name();
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/conf/TosKeys.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/conf/TosKeys.java
new file mode 100644
index 00000000000..33b4055ef60
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/conf/TosKeys.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.conf;
+
+import org.apache.hadoop.fs.tosfs.object.ChecksumType;
+import org.apache.hadoop.fs.tosfs.object.tos.TOSErrorCodes;
+
+public class TosKeys {
+
+ /**
+ * The accessKey key to access the tos object storage.
+ */
+ public static final String FS_TOS_ACCESS_KEY_ID = "fs.tos.access-key-id";
+
+ /**
+ * The secret access key to access the object storage.
+ */
+ public static final String FS_TOS_SECRET_ACCESS_KEY = "fs.tos.secret-access-key";
+
+ /**
+ * The session token to access the object storage.
+ */
+ public static final String FS_TOS_SESSION_TOKEN = "fs.tos.session-token";
+
+ /**
+ * The access key to access the object storage for the configured bucket, where %s is the bucket
+ * name.
+ */
+ public static final ArgumentKey FS_TOS_BUCKET_ACCESS_KEY_ID =
+ new ArgumentKey("fs.tos.bucket.%s.access-key-id");
+
+ /**
+ * The secret access key to access the object storage for the configured bucket, where %s is the
+ * bucket name.
+ */
+ public static final ArgumentKey FS_TOS_BUCKET_SECRET_ACCESS_KEY =
+ new ArgumentKey("fs.tos.bucket.%s.secret-access-key");
+
+ /**
+ * The session token to access the object storage for the configured bucket, where %s is the
+ * bucket name.
+ */
+ public static final ArgumentKey FS_TOS_BUCKET_SESSION_TOKEN =
+ new ArgumentKey("fs.tos.bucket.%s.session-token");
+
+ // Credential
+ /**
+ * Default credentials provider chain that looks for credentials in this order:
+ * SimpleCredentialsProvider,EnvironmentCredentialsProvider.
+ */
+ public static final String FS_TOS_CREDENTIALS_PROVIDER = "fs.tos.credentials.provider";
+ public static final String FS_TOS_CREDENTIALS_PROVIDER_DEFAULT =
+ "org.apache.hadoop.fs.tosfs.object.tos.auth.DefaultCredentialsProviderChain";
+
+ /**
+ * User customized credential provider classes, separate provider class name with comma if there
+ * are multiple providers.
+ */
+ public static final String FS_TOS_CUSTOM_CREDENTIAL_PROVIDER_CLASSES =
+ "fs.tos.credential.provider.custom.classes";
+
+ public static final String[] FS_TOS_CUSTOM_CREDENTIAL_PROVIDER_CLASSES_DEFAULT =
+ new String[] { "org.apache.hadoop.fs.tosfs.object.tos.auth.EnvironmentCredentialsProvider",
+ "org.apache.hadoop.fs.tosfs.object.tos.auth.SimpleCredentialsProvider" };
+
+ /**
+ * Set a canned ACL for newly created and copied objects. Value may be 'private', 'public-read',
+ * 'public-read-write', 'authenticated-read', 'bucket-owner-read', 'bucket-owner-full-control',
+ * 'bucket-owner-entrusted'. If set, caller IAM role must have 'tos:PutObjectAcl' permission on
+ * the bucket.
+ */
+ public static final String FS_TOS_ACL_DEFAULT = "fs.tos.acl.default";
+
+ // TOS http client.
+ /**
+ * The maximum number of connections to the TOS service that a client can create.
+ */
+ public static final String FS_TOS_HTTP_MAX_CONNECTIONS = "fs.tos.http.maxConnections";
+ public static final int FS_TOS_HTTP_MAX_CONNECTIONS_DEFAULT = 1024;
+
+ /**
+ * The time that a connection thread can be in idle state, larger than which the thread will be
+ * terminated.
+ */
+ public static final String FS_TOS_HTTP_IDLE_CONNECTION_TIME_MILLS =
+ "fs.tos.http.idleConnectionTimeMills";
+ public static final int FS_TOS_HTTP_IDLE_CONNECTION_TIME_MILLS_DEFAULT = 60000;
+
+ /**
+ * The connect timeout that the tos client tries to connect to the TOS service.
+ */
+ public static final String FS_TOS_HTTP_CONNECT_TIMEOUT_MILLS = "fs.tos.http.connectTimeoutMills";
+ public static final int FS_TOS_HTTP_CONNECT_TIMEOUT_MILLS_DEFAULT = 10000;
+
+ /**
+ * The reading timeout when reading data from tos. Note that it is configured for the tos client
+ * sdk, not hadoop-tos.
+ */
+ public static final String FS_TOS_HTTP_READ_TIMEOUT_MILLS = "fs.tos.http.readTimeoutMills";
+ public static final int FS_TOS_HTTP_READ_TIMEOUT_MILLS_DEFAULT = 30000;
+
+ /**
+ * The writing timeout when uploading data to tos. Note that it is configured for the tos client
+ * sdk, not hadoop-tos.
+ */
+ public static final String FS_TOS_HTTP_WRITE_TIMEOUT_MILLS = "fs.tos.http.writeTimeoutMills";
+ public static final int FS_TOS_HTTP_WRITE_TIMEOUT_MILLS_DEFAULT = 30000;
+
+ /**
+ * Enables SSL connections to TOS or not.
+ */
+ public static final String FS_TOS_HTTP_ENABLE_VERIFY_SSL = "fs.tos.http.enableVerifySSL";
+ public static final boolean FS_TOS_HTTP_ENABLE_VERIFY_SSL_DEFAULT = true;
+
+ /**
+ * The timeout (in minutes) of the dns cache used in tos client.
+ */
+ public static final String FS_TOS_HTTP_DNS_CACHE_TIME_MINUTES = "fs.tos.http.dnsCacheTimeMinutes";
+ public static final int FS_TOS_HTTP_DNS_CACHE_TIME_MINUTES_DEFAULT = 0;
+
+ /**
+ * Used for directory bucket, whether enable recursive delete capability in TOS server, which will
+ * atomic delete all objects under given dir(inclusive), otherwise the client will list all sub
+ * objects, and then send batch delete request to TOS to delete dir.
+ */
+ public static final String FS_TOS_RMR_SERVER_ENABLED = "fs.tos.rmr.server.enabled";
+ public static final boolean FS_FS_TOS_RMR_SERVER_ENABLED_DEFAULT = false;
+
+ /**
+ * If fs.tos.rmr.client.enabled is true, client will list all objects under the given dir and
+ * delete them by batch. Set value with true will use the recursive delete capability of TOS SDK,
+ * otherwise will delete object one by one via preorder tree walk.
+ */
+ public static final String FS_TOS_RMR_CLIENT_ENABLE = "fs.tos.rmr.client.enabled";
+ public static final boolean FS_TOS_RMR_CLIENT_ENABLE_DEFAULT = true;
+
+ /**
+ * The prefix will be used as the product name in TOS SDK. The final user agent pattern is
+ * '{prefix}/TOS_FS/{hadoop tos version}'.
+ */
+ public static final String FS_TOS_USER_AGENT_PREFIX = "fs.tos.user.agent.prefix";
+ public static final String FS_TOS_USER_AGENT_PREFIX_DEFAULT = "HADOOP-TOS";
+
+ // TOS common keys.
+ /**
+ * The threshold indicates whether reuse the socket connection to optimize read performance during
+ * closing tos object inputstream of get object. If the remaining bytes is less than max drain
+ * bytes during closing the inputstream, will just skip the bytes instead of closing the socket
+ * connection.
+ */
+ public static final String FS_TOS_MAX_DRAIN_BYTES = "fs.tos.max-drain-bytes";
+ public static final long FS_TOS_MAX_DRAIN_BYTES_DEFAULT = 1024 * 1024L;
+
+ /**
+ * Whether disable the tos http client cache in the current JVM.
+ */
+ public static final String FS_TOS_DISABLE_CLIENT_CACHE = "fs.tos.client.disable.cache";
+ public static final boolean FS_TOS_DISABLE_CLIENT_CACHE_DEFAULT = false;
+
+ /**
+ * The batch size when deleting the objects in batches.
+ */
+ public static final String FS_TOS_DELETE_OBJECTS_COUNT = "fs.tos.batch.delete.objects-count";
+ public static final int FS_TOS_DELETE_OBJECTS_COUNT_DEFAULT = 1000;
+
+ /**
+ * The maximum retry times when deleting objects in batches failed.
+ */
+ public static final String FS_TOS_BATCH_DELETE_MAX_RETRIES = "fs.tos.batch.delete.max-retries";
+ public static final int FS_TOS_BATCH_DELETE_MAX_RETRIES_DEFAULT = 20;
+
+ /**
+ * The codes from TOS deleteMultiObjects response, client will resend the batch delete request to
+ * delete the failed keys again if the response only contains these codes, otherwise won't send
+ * request anymore.
+ */
+ public static final String FS_TOS_BATCH_DELETE_RETRY_CODES = "fs.tos.batch.delete.retry-codes";
+ public static final String[] FS_TOS_BATCH_DELETE_RETRY_CODES_DEFAULT =
+ new String[] { "ExceedAccountQPSLimit", "ExceedAccountRateLimit", "ExceedBucketQPSLimit",
+ "ExceedBucketRateLimit", "InternalError", "ServiceUnavailable", "SlowDown",
+ "TooManyRequests" };
+
+ /**
+ * The retry interval (in milliseconds) when deleting objects in batches failed.
+ */
+ public static final String FS_TOS_BATCH_DELETE_RETRY_INTERVAL =
+ "fs.tos.batch.delete.retry.interval";
+ public static final long FS_TOS_BATCH_DELETE_RETRY_INTERVAL_DEFAULT = 1000L;
+
+ /**
+ * The batch size of listing object per request for the given object storage, such as listing a
+ * directory, searching for all objects whose path starts with the directory path, and returning
+ * them as a list.
+ */
+ public static final String FS_TOS_LIST_OBJECTS_COUNT = "fs.tos.list.objects-count";
+ public static final int FS_TOS_LIST_OBJECTS_COUNT_DEFAULT = 1000;
+
+ /**
+ * The maximum retry times of sending request via TOS client, client will resend the request if
+ * got retryable exceptions, e.g. SocketException, UnknownHostException, SSLException,
+ * InterruptedException, SocketTimeoutException, or got TOO_MANY_REQUESTS, INTERNAL_SERVER_ERROR
+ * http codes.
+ */
+ public static final String FS_TOS_REQUEST_MAX_RETRY_TIMES = "fs.tos.request.max.retry.times";
+ public static final int FS_TOS_REQUEST_MAX_RETRY_TIMES_DEFAULT = 20;
+
+ /**
+ * The fast-fail error codes means the error cannot be solved by retrying the request. TOS client
+ * won't retry the request if receiving a 409 http status code and if the error code is in the
+ * configured non-retryable error code list.
+ */
+ public static final String FS_TOS_FAST_FAILURE_409_ERROR_CODES =
+ "fs.tos.fast-fail-409-error-codes";
+ public static final String FS_TOS_FAST_FAILURE_409_ERROR_CODES_DEFAULT =
+ TOSErrorCodes.FAST_FAILURE_CONFLICT_ERROR_CODES;
+
+ /**
+ * The maximum retry times of reading object content via TOS client, client will resend the
+ * request to create a new input stream if getting unexpected end of stream error during reading
+ * the input stream.
+ */
+ public static final String FS_TOS_MAX_READ_OBJECT_RETRIES = "fs.tos.inputstream.max.retry.times";
+ public static final int FS_TOS_MAX_READ_OBJECT_RETRIES_DEFAULT = 5;
+
+ /**
+ * Enable the crc check when uploading files to tos or not.
+ */
+ public static final String FS_TOS_CRC_CHECK_ENABLED = "fs.tos.crc.check.enable";
+ public static final boolean FS_TOS_CRC_CHECK_ENABLED_DEFAULT = true;
+
+ /**
+ * Whether enable tos getFileStatus API or not, which returns the object info directly in one RPC
+ * request, otherwise, might need to send three RPC requests to get object info.
+ * For example, there is a key 'a/b/c' exists in TOS, and we want to get object status of 'a/b',
+ * the GetFileStatus('a/b') will return the prefix 'a/b/' as a directory object directly. If this
+ * property is disabled, we need to head('a/b') at first, and then head('a/b/'), and last call
+ * list('a/b/', limit=1) to get object info. Using GetFileStatus API can reduce the RPC call
+ * times.
+ */
+ public static final String FS_TOS_GET_FILE_STATUS_ENABLED = "fs.tos.get-file-status.enabled";
+ public static final boolean FS_TOS_GET_FILE_STATUS_ENABLED_DEFAULT = true;
+
+ /**
+ * The key indicates the name of the tos checksum algorithm. Specify the algorithm name to compare
+ * checksums between different storage systems. For example to compare checksums between hdfs and
+ * tos, we need to configure the algorithm name to COMPOSITE-CRC32C.
+ */
+ public static final String FS_TOS_CHECKSUM_ALGORITHM = "fs.tos.checksum-algorithm";
+ public static final String FS_TOS_CHECKSUM_ALGORITHM_DEFAULT = "TOS-CHECKSUM";
+
+ /**
+ * The key indicates how to retrieve file checksum from tos, error will be thrown if the
+ * configured checksum type is not supported by tos. The supported checksum types are:
+ * CRC32C, CRC64ECMA.
+ */
+ public static final String FS_TOS_CHECKSUM_TYPE = "fs.tos.checksum-type";
+ public static final String FS_TOS_CHECKSUM_TYPE_DEFAULT = ChecksumType.CRC64ECMA.name();
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/BucketInfo.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/BucketInfo.java
new file mode 100644
index 00000000000..8c9a9d6e152
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/BucketInfo.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
+
+import java.util.Objects;
+
+/**
+ * There are two kinds of bucket types: general purpose bucket(common bucket) and directory bucket.
+ * Directory bucket organize data hierarchically into directories as opposed to the flat storage
+ * structure of general purpose buckets.
+ * <p>
+ * Only a few object storages support directory bucket, e.g. S3, OBS, TOS. Generally, directory
+ * bucket supports rename or delete dir with constant time complexity O(1), but these object
+ * storages have slight differences on these APIs. E.g. S3 doesn't provide rename API. S3 will
+ * recursively delete any empty parent directories in the object path during delete an object in a
+ * directory bucket, but TOS won't delete any empty parent directories.
+ * <p>
+ * And also there are some general difference between general purpose bucket and directory bucket.
+ * <ul>
+ * <li>Directory bucket treats the object end with '/' as the directory during creating object.
+ * </li>
+ * <li>TOS directory bucket will create missed parent dir during create object automatically,
+ * but general purpose bucket only create one object.</li>
+ * <li>Directory bucket doesn't allow create any object under a file, but general purpose bucket
+ * haven't this constraint.</li>
+ * <li>If a object 'a/b/' exists in directory bucket, both head('a/b') and head('a/b/') will get
+ * the object meta, but only head('a/b/') can get the object meta from general purpose bucket</li>
+ * <li>TOS directory bucket provides atomic rename/delete dir abilities</li>
+ * </ul>
+ */
+public class BucketInfo {
+ private final String name;
+ private final boolean directory;
+
+ public BucketInfo(String name, boolean directory) {
+ this.name = name;
+ this.directory = directory;
+ }
+
+ public String name() {
+ return name;
+ }
+
+ public boolean isDirectory() {
+ return directory;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ } else if (!(o instanceof BucketInfo)) {
+ return false;
+ }
+
+ BucketInfo that = (BucketInfo) o;
+ return Objects.equals(name, that.name)
+ && Objects.equals(directory, that.directory);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, directory);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("name", name)
+ .add("directory", directory)
+ .toString();
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ChecksumInfo.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ChecksumInfo.java
new file mode 100644
index 00000000000..b7e529d3c11
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ChecksumInfo.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+public class ChecksumInfo {
+ private final String algorithm;
+ private final ChecksumType checksumType;
+
+ public ChecksumInfo(String algorithm, ChecksumType checksumType) {
+ this.algorithm = algorithm;
+ this.checksumType = checksumType;
+ }
+
+ public String algorithm() {
+ return algorithm;
+ }
+
+ public ChecksumType checksumType() {
+ return checksumType;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ChecksumType.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ChecksumType.java
new file mode 100644
index 00000000000..6654e11496b
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ChecksumType.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+public enum ChecksumType {
+ NULL((byte) 0, 0),
+ CRC32((byte) 1, 4),
+
+ CRC32C((byte) 2, 4),
+ CRC64ECMA((byte) 3, 8),
+ MD5((byte) 4, 128);
+
+ private final byte value;
+ private final int bytes;
+
+ ChecksumType(byte value, int bytes) {
+ this.value = value;
+ this.bytes = bytes;
+ }
+
+ public byte value() {
+ return value;
+ }
+
+ public int bytes() {
+ return bytes;
+ }
+
+ public static ChecksumType valueOf(byte value) {
+ for (ChecksumType t : values()) {
+ if (t.value == value) {
+ return t;
+ }
+ }
+ throw new IllegalStateException("Unknown checksum type with value: " + value);
+ }
+
+ public static int maxBytes() {
+ int maxBytes = 0;
+ for (ChecksumType t : values()) {
+ maxBytes = Math.max(maxBytes, t.bytes());
+ }
+ return maxBytes;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/Constants.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/Constants.java
new file mode 100644
index 00000000000..a8aa3fe686d
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/Constants.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+public class Constants {
+ private Constants() {
+ }
+
+ public static final String TOS_FS = "TOS_FS";
+
+ // Magic checksum means doesn't support checksum, if the file type is dir or the filesystem/object
+ // storage doesn't implement checksum algorithm will use magic checksum as the file checksum.
+ public static final byte[] MAGIC_CHECKSUM = new byte[] { 'M' };
+
+ public static final String SLASH = "/";
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/DirectoryStorage.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/DirectoryStorage.java
new file mode 100644
index 00000000000..d840c9842a0
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/DirectoryStorage.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+public interface DirectoryStorage extends ObjectStorage {
+
+ /**
+ * List the objects under the given dir key. Does not guarantee to return the list of objects in a
+ * sorted order.
+ *
+ * @param key the dir key.
+ * @param recursive indicate whether list dir recursively or not.
+ * @return the sub objects of the given dir key, not include the dir key.
+ */
+ Iterable<ObjectInfo> listDir(String key, boolean recursive);
+
+ /**
+ * The given key could be a file or a directory, if a directory, it can an empty or non-empty
+ * directory.
+ * Deleting a non-empty dir is only allowed if recursive is true.
+ *
+ * @param key the dir key.
+ * @param recursive indicate whether delete dir recursively or not.
+ */
+ void deleteDir(String key, boolean recursive);
+
+ /**
+ * Whether the directory is empty.
+ *
+ * @param key the dir key.
+ * @return true if the dir is empty or doesn't exist.
+ */
+ boolean isEmptyDir(String key);
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/FileStore.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/FileStore.java
new file mode 100644
index 00000000000..b1a4141b7d5
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/FileStore.java
@@ -0,0 +1,632 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.conf.FileStoreKeys;
+import org.apache.hadoop.fs.tosfs.object.exceptions.NotAppendableException;
+import org.apache.hadoop.fs.tosfs.object.request.ListObjectsRequest;
+import org.apache.hadoop.fs.tosfs.object.response.ListObjectsResponse;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.fs.tosfs.util.Range;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class FileStore implements ObjectStorage {
+
+ private static final Logger LOG = LoggerFactory.getLogger(FileStore.class);
+
+ private static final String NAME = "filestore";
+ private static final String STAGING_DIR = "__STAGING__";
+ private static final String SLASH = "/";
+
+ public static final String DEFAULT_BUCKET = "dummy-bucket";
+ public static final String ENV_FILE_STORAGE_ROOT = "FILE_STORAGE_ROOT";
+
+ private static final int MAX_DELETE_OBJECTS_COUNT = 1000;
+ private static final int MIN_PART_SIZE = 5 * 1024 * 1024;
+ private static final int MAX_PART_COUNT = 10000;
+
+ private String bucket;
+ private String root;
+ private Configuration conf;
+ private ChecksumInfo checksumInfo;
+
+ @Override
+ public String scheme() {
+ return NAME;
+ }
+
+ @Override
+ public BucketInfo bucket() {
+ return new BucketInfo(bucket, false);
+ }
+
+ @Override
+ public void initialize(Configuration conf, String bucket) {
+ this.bucket = bucket;
+ this.conf = conf;
+ String endpoint = conf.get(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key(NAME));
+ if (endpoint == null || endpoint.isEmpty()) {
+ endpoint = System.getenv(ENV_FILE_STORAGE_ROOT);
+ }
+ Preconditions.checkNotNull(endpoint, "%s cannot be null",
+ ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key(NAME));
+
+ if (endpoint.endsWith(SLASH)) {
+ this.root = endpoint;
+ } else {
+ this.root = endpoint + SLASH;
+ }
+ LOG.debug("the root path is: {}", this.root);
+
+ String algorithm = conf.get(FileStoreKeys.FS_FILESTORE_CHECKSUM_ALGORITHM,
+ FileStoreKeys.FS_FILESTORE_CHECKSUM_ALGORITHM_DEFAULT);
+ ChecksumType checksumType = ChecksumType.valueOf(conf.get(FileStoreKeys.FS_FILESTORE_CHECKSUM_TYPE,
+ FileStoreKeys.FS_FILESTORE_CHECKSUM_TYPE_DEFAULT).toUpperCase());
+ Preconditions.checkArgument(checksumType == ChecksumType.MD5,
+ "Checksum type %s is not supported by FileStore.", checksumType.name());
+ checksumInfo = new ChecksumInfo(algorithm, checksumType);
+
+ new File(root).mkdirs();
+ }
+
+ @Override
+ public Configuration conf() {
+ return conf;
+ }
+
+ private static String encode(String key) {
+ try {
+ return URLEncoder.encode(key, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ LOG.warn("failed to encode key: {}", key);
+ return key;
+ }
+ }
+
+ private static String decode(String key) {
+ try {
+ return URLDecoder.decode(key, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ LOG.warn("failed to decode key: {}", key);
+ return key;
+ }
+ }
+
+ @Override
+ public ObjectContent get(String key, long offset, long limit) {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "Key should not be empty.");
+ File file = path(encode(key)).toFile();
+ if (!file.exists()) {
+ throw new RuntimeException(String.format("File not found %s", file.getAbsolutePath()));
+ }
+
+ Range range = ObjectUtils.calculateRange(offset, limit, file.length());
+ try (FileInputStream in = new FileInputStream(file)) {
+ in.skip(range.off());
+ byte[] bs = new byte[(int) range.len()];
+ in.read(bs);
+
+ byte[] fileChecksum = getFileChecksum(file.toPath());
+ return new ObjectContent(fileChecksum, new ByteArrayInputStream(bs));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public byte[] put(String key, InputStreamProvider streamProvider, long contentLength) {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "Key should not be empty.");
+ File destFile = path(encode(key)).toFile();
+ copyInputStreamToFile(streamProvider.newStream(), destFile, contentLength);
+
+ return ObjectInfo.isDir(key) ? Constants.MAGIC_CHECKSUM : getFileChecksum(destFile.toPath());
+ }
+
+ @Override
+ public byte[] append(String key, InputStreamProvider streamProvider, long contentLength) {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "Key should not be empty.");
+ File destFile = path(encode(key)).toFile();
+ if (!destFile.exists()) {
+ if (contentLength == 0) {
+ throw new NotAppendableException(String.format("%s is not appendable because append non-existed object with "
+ + "zero byte is not supported.", key));
+ }
+ return put(key, streamProvider, contentLength);
+ } else {
+ appendInputStreamToFile(streamProvider.newStream(), destFile, contentLength);
+ return ObjectInfo.isDir(key) ? Constants.MAGIC_CHECKSUM : getFileChecksum(destFile.toPath());
+ }
+ }
+
+ private static File createTmpFile(File destFile) {
+ String tmpFilename = ".tmp." + UUIDUtils.random();
+ File file = new File(destFile.getParentFile(), tmpFilename);
+
+ try {
+ if (!file.exists() && !file.createNewFile()) {
+ throw new RuntimeException("failed to create tmp file");
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return file;
+ }
+
+ @Override
+ public void delete(String key) {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "Key should not be empty.");
+ File file = path(encode(key)).toFile();
+ if (file.exists()) {
+ try {
+ if (file.isDirectory()) {
+ FileUtils.deleteDirectory(file);
+ } else {
+ Files.delete(file.toPath());
+ }
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+ @Override
+ public List<String> batchDelete(List<String> keys) {
+ Preconditions.checkArgument(keys.size() <= MAX_DELETE_OBJECTS_COUNT,
+ "The batch delete object count should <= %s", MAX_DELETE_OBJECTS_COUNT);
+ List<String> failedKeys = Lists.newArrayList();
+ for (String key : keys) {
+ try {
+ delete(key);
+ } catch (Exception e) {
+ LOG.error("Failed to delete key {}", key, e);
+ failedKeys.add(key);
+ }
+ }
+ return failedKeys;
+ }
+
+ @Override
+ public void deleteAll(String prefix) {
+ Iterable<ObjectInfo> objects = listAll(prefix, "");
+ ObjectUtils.deleteAllObjects(this, objects, conf.getInt(ConfKeys.FS_BATCH_DELETE_SIZE.key(NAME),
+ ConfKeys.FS_BATCH_DELETE_SIZE_DEFAULT));
+ }
+
+ @Override
+ public ObjectInfo head(String key) {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "Key should not be empty.");
+ File file = path(encode(key)).toFile();
+ if (file.exists()) {
+ return toObjectInfo(file.toPath());
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public Iterable<ListObjectsResponse> list(ListObjectsRequest request) {
+ try (Stream<Path> stream = Files.walk(Paths.get(root))) {
+ List<ObjectInfo> allObjects = list(stream, request.prefix(), request.startAfter())
+ .collect(Collectors.toList());
+ int maxKeys = request.maxKeys() < 0 ? allObjects.size() : request.maxKeys();
+ return Collections.singletonList(
+ splitObjects(request.prefix(), request.delimiter(), maxKeys, request.startAfter(), allObjects));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private ListObjectsResponse splitObjects(
+ String prefix,
+ String delimiter,
+ int limit,
+ String startAfter,
+ List<ObjectInfo> objects) {
+ int retSize = Math.min(limit, objects.size());
+
+ if (Strings.isNullOrEmpty(delimiter)) {
+ // the response only contains objects
+ List<ObjectInfo> retObjs = objects.subList(0, retSize);
+ return new ListObjectsResponse(retObjs, Collections.emptyList());
+ } else {
+ // the response only contains objects and common prefixes
+ Set<String> commonPrefixes = new TreeSet<>();
+ List<ObjectInfo> objectInfos = new ArrayList<>();
+
+ for (ObjectInfo obj : objects) {
+ String suffixKey = obj.key().substring(prefix.length());
+ String[] tokens = suffixKey.split(delimiter, 2);
+ if (tokens.length == 2) {
+ String key = prefix + tokens[0] + delimiter;
+ // the origin key is bigger than startAfter,
+ // but after the new key after split might equal to startAfter, need to exclude.
+ if (!key.equals(startAfter)) {
+ commonPrefixes.add(key);
+
+ // why don't break the loop before add new key to common prefixes list
+ // is that the new key might be an existed common prefix, but we still want to continue visited new object
+ // since the new object might also be an existed common prefix until the total size is out of limit.
+ if (commonPrefixes.size() + objectInfos.size() > retSize) {
+ commonPrefixes.remove(key);
+ break;
+ }
+ }
+ } else {
+ if (commonPrefixes.size() + objectInfos.size() >= retSize) {
+ break;
+ }
+
+ objectInfos.add(obj);
+ }
+ }
+ return new ListObjectsResponse(objectInfos, new ArrayList<>(commonPrefixes));
+ }
+ }
+
+ @Override
+ public MultipartUpload createMultipartUpload(String key) {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "Key should not be empty.");
+
+ String uploadId = UUIDUtils.random();
+ Path uploadDir = uploadPath(key, uploadId);
+ if (uploadDir.toFile().mkdirs()) {
+ return new MultipartUpload(key, uploadId, MIN_PART_SIZE, MAX_PART_COUNT);
+ } else {
+ throw new RuntimeException("Failed to create MultipartUpload with key: " + key);
+ }
+ }
+
+ private Path uploadPath(String key, String uploadId) {
+ return Paths.get(root, STAGING_DIR, encode(key), uploadId);
+ }
+
+ @Override
+ public Part uploadPart(
+ String key, String uploadId, int partNum,
+ InputStreamProvider streamProvider, long contentLength) {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "Key should not be empty.");
+
+ File uploadDir = uploadPath(key, uploadId).toFile();
+ if (!uploadDir.exists()) {
+ throw new RuntimeException("cannot locate the upload id: " + uploadId);
+ }
+
+ File partFile = new File(uploadDir, String.valueOf(partNum));
+ copyInputStreamToFile(streamProvider.newStream(), partFile, contentLength);
+
+ try {
+ byte[] data = Files.readAllBytes(partFile.toPath());
+ return new Part(partNum, data.length, DigestUtils.md5Hex(data));
+ } catch (IOException e) {
+ LOG.error("failed to locate the part file: {}", partFile.getAbsolutePath());
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static void appendInputStreamToFile(InputStream in, File partFile, long contentLength) {
+ try (FileOutputStream out = new FileOutputStream(partFile, true)) {
+ long copiedBytes = IOUtils.copyLarge(in, out, 0, contentLength);
+
+ if (copiedBytes < contentLength) {
+ throw new IOException(String.format("Unexpect end of stream, expected to write length:%s, actual written:%s",
+ contentLength, copiedBytes));
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ CommonUtils.runQuietly(in::close);
+ }
+ }
+
+ private static void copyInputStreamToFile(InputStream in, File partFile, long contentLength) {
+ File tmpFile = createTmpFile(partFile);
+ try (FileOutputStream out = new FileOutputStream(tmpFile)) {
+ long copiedBytes = IOUtils.copyLarge(in, out, 0, contentLength);
+
+ if (copiedBytes < contentLength) {
+ throw new IOException(String.format("Unexpect end of stream, expected length:%s, actual:%s",
+ contentLength, tmpFile.length()));
+ }
+ } catch (IOException e) {
+ CommonUtils.runQuietly(() -> FileUtils.delete(tmpFile));
+ throw new RuntimeException(e);
+ } finally {
+ CommonUtils.runQuietly(in::close);
+ }
+
+ if (!tmpFile.renameTo(partFile)) {
+ throw new RuntimeException("failed to put file since rename fail.");
+ }
+ }
+
+ @Override
+ public byte[] completeUpload(String key, String uploadId, List<Part> uploadParts) {
+ Preconditions.checkArgument(uploadParts != null && uploadParts.size() > 0, "upload parts cannot be null or empty.");
+ File uploadDir = uploadPath(key, uploadId).toFile();
+ if (!uploadDir.exists()) {
+ throw new RuntimeException("cannot locate the upload id: " + uploadId);
+ }
+
+ List<Integer> partNums = listPartNums(uploadDir);
+ if (partNums.size() != uploadParts.size()) {
+ throw new RuntimeException(String.format("parts length mismatched: %d != %d",
+ partNums.size(), uploadParts.size()));
+ }
+
+ Collections.sort(partNums);
+ uploadParts.sort(Comparator.comparingInt(Part::num));
+
+ Path keyPath = path(encode(key));
+ File tmpFile = createTmpFile(keyPath.toFile());
+ try (FileOutputStream outputStream = new FileOutputStream(tmpFile);
+ FileChannel outputChannel = outputStream.getChannel()) {
+ int offset = 0;
+ for (int i = 0; i < partNums.size(); i++) {
+ Part part = uploadParts.get(i);
+ if (part.num() != partNums.get(i)) {
+ throw new RuntimeException(String.format("part num mismatched: %d != %d", part.num(), partNums.get(i)));
+ }
+
+ File partFile = new File(uploadDir, String.valueOf(part.num()));
+ checkPartFile(part, partFile);
+
+ try (FileInputStream inputStream = new FileInputStream(partFile);
+ FileChannel inputChannel = inputStream.getChannel()) {
+ outputChannel.transferFrom(inputChannel, offset, partFile.length());
+ offset += partFile.length();
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ if (!tmpFile.renameTo(keyPath.toFile())) {
+ throw new RuntimeException("rename file failed");
+ } else {
+ try {
+ FileUtils.deleteDirectory(uploadDir);
+ } catch (IOException e) {
+ LOG.warn("failed to clean upload directory.");
+ }
+ }
+
+ return getFileChecksum(keyPath);
+ }
+
+ private byte[] getFileChecksum(Path keyPath) {
+ return getFileMD5(keyPath);
+ }
+
+ private static byte[] getFileMD5(Path keyPath) {
+ try {
+ return DigestUtils.md5(Files.readAllBytes(keyPath));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static void checkPartFile(Part part, File partFile) throws IOException {
+ if (part.size() != partFile.length()) {
+ throw new RuntimeException(String.format("part size mismatched: %d != %d",
+ part.size(), partFile.length()));
+ }
+
+ try (FileInputStream inputStream = new FileInputStream(partFile)) {
+ String md5Hex = DigestUtils.md5Hex(inputStream);
+ if (!Objects.equals(part.eTag(), md5Hex)) {
+ throw new RuntimeException(String.format("part etag mismatched: %s != %s",
+ part.eTag(), md5Hex));
+ }
+ }
+ }
+
+ private List<Integer> listPartNums(File uploadDir) {
+ try (Stream<Path> stream = Files.list(uploadDir.toPath())) {
+ return stream
+ .map(f -> Integer.valueOf(f.toFile().getName()))
+ .collect(Collectors.toList());
+ } catch (IOException e) {
+ LOG.error("failed to list part files.");
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void abortMultipartUpload(String key, String uploadId) {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "Key should not be empty.");
+ Path uploadDir = uploadPath(key, uploadId);
+ if (uploadDir.toFile().exists()) {
+ try {
+ FileUtils.deleteDirectory(uploadDir.toFile());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public Iterable<MultipartUpload> listUploads(String prefix) {
+ Path stagingDir = Paths.get(root, STAGING_DIR);
+ if (!Files.exists(stagingDir)) {
+ return Collections.emptyList();
+ }
+ try (Stream<Path> encodedKeyStream = Files.list(stagingDir)) {
+ return encodedKeyStream
+ .filter(key -> Objects.equals(prefix, "") || key.toFile().getName().startsWith(encode(prefix)))
+ .flatMap(key -> {
+ try {
+ return Files.list(key)
+ .map(id -> new MultipartUpload(decode(key.toFile().getName()),
+ id.toFile().getName(), MIN_PART_SIZE, MAX_PART_COUNT));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .sorted()
+ .collect(Collectors.toList());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Part uploadPartCopy(
+ String srcKey, String dstKey, String uploadId, int partNum, long copySourceRangeStart,
+ long copySourceRangeEnd) {
+ File uploadDir = uploadPath(dstKey, uploadId).toFile();
+ if (!uploadDir.exists()) {
+ throw new RuntimeException(String.format("Upload directory %s already exits", uploadDir));
+ }
+ File partFile = new File(uploadDir, String.valueOf(partNum));
+ int fileSize = (int) (copySourceRangeEnd - copySourceRangeStart + 1);
+ try (InputStream is = get(srcKey, copySourceRangeStart, fileSize).stream();
+ FileOutputStream fos = new FileOutputStream(partFile)) {
+ byte[] data = new byte[fileSize];
+ IOUtils.readFully(is, data);
+ fos.write(data);
+ return new Part(partNum, fileSize, DigestUtils.md5Hex(data));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void copy(String srcKey, String dstKey) {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(srcKey), "Src key should not be empty.");
+ File file = path(encode(srcKey)).toFile();
+ if (!file.exists()) {
+ throw new RuntimeException(String.format("File not found %s", file.getAbsolutePath()));
+ }
+
+ put(dstKey, () -> get(srcKey).stream(), file.length());
+ }
+
+ @Override
+ public void rename(String srcKey, String dstKey) {
+ Preconditions.checkArgument(!Objects.equals(srcKey, dstKey), "Cannot rename to the same object");
+ Preconditions.checkNotNull(head(srcKey), "Source key %s doesn't exist", srcKey);
+
+ File srcFile = path(encode(srcKey)).toFile();
+ File dstFile = path(encode(dstKey)).toFile();
+ boolean ret = srcFile.renameTo(dstFile);
+ if (!ret) {
+ throw new RuntimeException(String.format("Failed to rename %s to %s", srcKey, dstKey));
+ }
+ }
+
+ @Override
+ public ObjectInfo objectStatus(String key) {
+ ObjectInfo obj = head(key);
+ if (obj == null && !ObjectInfo.isDir(key)) {
+ key = key + '/';
+ obj = head(key);
+ }
+
+ if (obj == null) {
+ Iterable<ObjectInfo> objs = list(key, null, 1);
+ if (objs.iterator().hasNext()) {
+ obj = new ObjectInfo(key, 0, new Date(0), Constants.MAGIC_CHECKSUM);
+ }
+ }
+
+ return obj;
+ }
+
+ @Override
+ public ChecksumInfo checksumInfo() {
+ return checksumInfo;
+ }
+
+ private Stream<ObjectInfo> list(Stream<Path> stream, String prefix, String startAfter) {
+ return stream
+ .filter(p -> {
+ String absolutePath = p.toFile().getAbsolutePath();
+ return !Objects.equals(key(absolutePath), "") &&
+ decode(key(absolutePath)).startsWith(prefix)
+ && !absolutePath.contains(STAGING_DIR)
+ && filter(decode(key(absolutePath)), startAfter);
+ })
+ .map(this::toObjectInfo)
+ .sorted(Comparator.comparing(ObjectInfo::key));
+ }
+
+ private boolean filter(String key, String startAfter) {
+ if (Strings.isNullOrEmpty(startAfter)) {
+ return true;
+ } else {
+ return key.compareTo(startAfter) > 0;
+ }
+ }
+
+ private ObjectInfo toObjectInfo(Path path) {
+ File file = path.toFile();
+ String key = decode(key(file.getAbsolutePath()));
+ return new ObjectInfo(key, file.length(), new Date(file.lastModified()), getFileChecksum(path));
+ }
+
+ private Path path(String key) {
+ return Paths.get(root, key);
+ }
+
+ private String key(String path) {
+ if (path.length() < root.length()) {
+ // root = path + "/"
+ return "";
+ }
+ return path.substring(root.length());
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/InputStreamProvider.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/InputStreamProvider.java
new file mode 100644
index 00000000000..98e654f99b5
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/InputStreamProvider.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import java.io.InputStream;
+
+/**
+ * Provides the content stream of a request.
+ * <p>
+ * Each call to the {@link #newStream()} method must result in a stream
+ * whose position is at the beginning of the content.
+ * Implementations may return a new stream or the same stream for each call.
+ * If returning a new stream, the implementation must ensure to {@code close()}
+ * and free any resources acquired by the previous stream.
+ */
+public interface InputStreamProvider {
+ InputStream newStream();
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/MultipartUpload.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/MultipartUpload.java
new file mode 100644
index 00000000000..e7c89f1f827
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/MultipartUpload.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
+
+import java.util.Objects;
+
+public class MultipartUpload implements Comparable<MultipartUpload> {
+ private final String key;
+ private final String uploadId;
+ private final int minPartSize;
+ private final int maxPartCount;
+
+ public MultipartUpload(String key, String uploadId, int minPartSize, int maxPartCount) {
+ this.key = key;
+ this.uploadId = uploadId;
+ this.minPartSize = minPartSize;
+ this.maxPartCount = maxPartCount;
+ }
+
+ public String key() {
+ return key;
+ }
+
+ public String uploadId() {
+ return uploadId;
+ }
+
+ public int minPartSize() {
+ return minPartSize;
+ }
+
+ public int maxPartCount() {
+ return maxPartCount;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ } else if (!(o instanceof MultipartUpload)) {
+ return false;
+ }
+
+ MultipartUpload that = (MultipartUpload) o;
+ if (!Objects.equals(key, that.key)) {
+ return false;
+ }
+ if (!Objects.equals(uploadId, that.uploadId)) {
+ return false;
+ }
+ if (!Objects.equals(minPartSize, that.minPartSize)) {
+ return false;
+ }
+ return Objects.equals(maxPartCount, that.maxPartCount);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(key, uploadId, minPartSize, maxPartCount);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("key", key)
+ .add("uploadId", uploadId)
+ .add("minPartSize", minPartSize)
+ .add("maxPartCount", maxPartCount)
+ .toString();
+ }
+
+ @Override
+ public int compareTo(MultipartUpload o) {
+ if (this == o) {
+ return 0;
+ } else if (o == null) {
+ return 1;
+ } else if (this.key.compareTo(o.key) == 0) {
+ return this.uploadId.compareTo(o.uploadId);
+ } else {
+ return this.key.compareTo(o.key);
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectConstants.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectConstants.java
new file mode 100644
index 00000000000..2fb2e978800
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectConstants.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+public class ObjectConstants {
+ public static final int MIN_PART_SIZE = 5 * 1024 * 1024;
+ public static final int MAX_PART_COUNT = 10000;
+
+ private ObjectConstants() {
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectContent.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectContent.java
new file mode 100644
index 00000000000..6961fd83bb7
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectContent.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.fs.tosfs.object.exceptions.ChecksumMismatchException;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+
+import java.io.InputStream;
+import java.util.Arrays;
+
+public class ObjectContent {
+ private final byte[] checksum;
+ private final InputStream stream;
+
+ public ObjectContent(byte[] checksum, InputStream stream) {
+ this.checksum = checksum;
+ this.stream = stream;
+ }
+
+ public InputStream stream() {
+ return stream;
+ }
+
+ public InputStream verifiedStream(byte[] expectedChecksum) throws ChecksumMismatchException {
+ if (!Arrays.equals(expectedChecksum, checksum)) {
+ CommonUtils.runQuietly(stream::close);
+ throw new ChecksumMismatchException(expectedChecksum, checksum);
+ }
+
+ return stream;
+ }
+
+ public byte[] checksum() {
+ return checksum;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectInfo.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectInfo.java
new file mode 100644
index 00000000000..ec8cbd8d282
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectInfo.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
+import org.apache.hadoop.util.StringUtils;
+
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Objects;
+
+import static org.apache.hadoop.util.Preconditions.checkArgument;
+
+public class ObjectInfo {
+ private final String key;
+ private final long size;
+ private final Date mtime;
+ private final boolean isDir;
+ private final byte[] checksum;
+
+ public ObjectInfo(String key, long size, Date mtime, byte[] checksum) {
+ this(key, size, mtime, checksum, ObjectInfo.isDir(key));
+ }
+
+ public ObjectInfo(String key, long size, Date mtime, byte[] checksum, boolean isDir) {
+ checkArgument(key != null, "Key is null");
+ checkArgument(size >= 0, "The size of key(%s) is negative", key);
+ checkArgument(mtime != null, "The modified time of key(%s) null.", key);
+ this.key = key;
+ this.size = size;
+ this.mtime = mtime;
+ this.isDir = isDir;
+ // checksum can be null since some object storage might not support checksum.
+ this.checksum = checksum == null || isDir ? Constants.MAGIC_CHECKSUM : checksum;
+ }
+
+ public String key() {
+ return key;
+ }
+
+ /**
+ * The size of directory object is 0;
+ *
+ * @return the size of object.
+ */
+ public long size() {
+ return isDir ? 0 : size;
+ }
+
+ public Date mtime() {
+ return mtime;
+ }
+
+ /**
+ * @return {@link Constants#MAGIC_CHECKSUM} if the object is a dir or the object storage
+ * doesn't support the given checksum type.
+ */
+ public byte[] checksum() {
+ return checksum;
+ }
+
+ public boolean isDir() {
+ return isDir;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ } else if (!(o instanceof ObjectInfo)) {
+ return false;
+ }
+
+ ObjectInfo that = (ObjectInfo) o;
+ return Objects.equals(key, that.key)
+ && Objects.equals(size, that.size)
+ && Objects.equals(mtime, that.mtime)
+ && Arrays.equals(checksum, that.checksum)
+ && Objects.equals(isDir, that.isDir);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(key, size, mtime, Arrays.hashCode(checksum), isDir);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("key", key)
+ .add("size", size)
+ .add("mtime", mtime)
+ .add("checksum", StringUtils.byteToHexString(checksum))
+ .add("isDir", isDir)
+ .toString();
+ }
+
+ public static boolean isDir(String key) {
+ return key.endsWith(Constants.SLASH);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectMultiRangeInputStream.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectMultiRangeInputStream.java
new file mode 100644
index 00000000000..4a9357e46bf
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectMultiRangeInputStream.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.fs.tosfs.util.FSUtils;
+import org.apache.hadoop.fs.tosfs.util.Range;
+import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
+import org.apache.hadoop.util.Preconditions;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class ObjectMultiRangeInputStream extends FSInputStream {
+ private final AtomicBoolean closed = new AtomicBoolean(false);
+ private final ExecutorService threadPool;
+ private final ObjectStorage storage;
+ private final String objectKey;
+ private final long contentLength;
+ private final long rangeSize;
+
+ private volatile ObjectRangeInputStream stream;
+ private volatile long nextPos = 0;
+ private volatile long currPos = 0;
+ // All range streams should have same checksum.
+ private final byte[] checksum;
+
+ public ObjectMultiRangeInputStream(
+ ExecutorService threadPool,
+ ObjectStorage storage,
+ Path path,
+ long contentLength,
+ long rangeSize,
+ byte[] checksum) {
+ this(threadPool, storage, ObjectUtils.pathToKey(path), contentLength, rangeSize, checksum);
+ }
+
+ public ObjectMultiRangeInputStream(
+ ExecutorService threadPool,
+ ObjectStorage storage,
+ String objectKey,
+ long contentLength,
+ long rangeSize,
+ byte[] checksum) {
+ this.threadPool = threadPool;
+ this.storage = storage;
+ this.objectKey = objectKey;
+ this.contentLength = contentLength;
+ this.rangeSize = rangeSize;
+ this.checksum = checksum;
+
+ Preconditions.checkNotNull(checksum, "Checksum should not be null.");
+ }
+
+ @Override
+ public synchronized void seek(long pos) throws IOException {
+ if (pos < 0) {
+ throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK + " " + pos);
+ }
+
+ if (contentLength <= 0) {
+ return;
+ }
+
+ nextPos = pos;
+ }
+
+ @Override
+ public synchronized long getPos() {
+ return nextPos;
+ }
+
+ @Override
+ public synchronized boolean seekToNewSource(long targetPos) throws IOException {
+ checkNotClosed();
+ return false;
+ }
+
+ @Override
+ public synchronized int read() throws IOException {
+ byte[] buf = new byte[1];
+ int n = read(buf, 0, buf.length);
+ if (n < 0) {
+ return -1;
+ } else {
+ return buf[0] & 0xFF;
+ }
+ }
+
+ @Override
+ public synchronized int read(byte[] buffer, int offset, int length) throws IOException {
+ checkNotClosed();
+ FSUtils.checkReadParameters(buffer, offset, length);
+ if (length == 0) {
+ return 0;
+ }
+
+ int total = 0;
+ while (total < length) {
+ if (contentLength == 0 || nextPos >= contentLength) {
+ return total == 0 ? -1 : total;
+ }
+
+ seekStream();
+ int n = stream.read(buffer, offset, length - total);
+ if (n < 0) {
+ return total == 0 ? -1 : total;
+ }
+
+ total += n;
+ offset += n;
+ currPos += n;
+ nextPos += n;
+ }
+
+ return total;
+ }
+
+ @Override
+ public int read(long position, byte[] buffer, int offset, int length) throws IOException {
+ checkNotClosed();
+ // Check the arguments, according to the HDFS contract.
+ if (position < 0) {
+ throw new EOFException("position is negative");
+ }
+ FSUtils.checkReadParameters(buffer, offset, length);
+ if (length == 0) {
+ return 0;
+ }
+
+ if (contentLength == 0 || position >= contentLength) {
+ return -1;
+ }
+
+ long remaining = contentLength - position;
+ int limit = (remaining >= length) ? length : (int) remaining;
+
+ try (InputStream in = storage.get(objectKey, position, limit).verifiedStream(checksum)) {
+ return in.read(buffer, offset, limit);
+ }
+ }
+
+ private void seekStream() throws IOException {
+ if (stream != null && stream.include(nextPos)) {
+ // Seek to a random position which is still located in the current range of stream.
+ if (nextPos != currPos) {
+ stream.seek(nextPos);
+ currPos = nextPos;
+ }
+ return;
+ }
+
+ // Seek to a position which is located in another range of new stream.
+ currPos = nextPos;
+ openStream();
+ }
+
+ private void openStream() throws IOException {
+ closeStream(true);
+
+ long off = (nextPos / rangeSize) * rangeSize;
+ Range range = Range.of(off, Math.min(contentLength - off, rangeSize));
+ if (nextPos < range.end()) {
+ stream = new ObjectRangeInputStream(storage, objectKey, range, checksum);
+ stream.seek(nextPos);
+ }
+ }
+
+ private void closeStream(boolean asyncClose) throws IOException {
+ if (stream != null) {
+ if (asyncClose) {
+ final ObjectRangeInputStream streamToClose = stream;
+ threadPool.submit(() -> CommonUtils.runQuietly(streamToClose::close));
+ } else {
+ stream.close();
+ }
+ stream = null;
+ }
+ }
+
+ private void checkNotClosed() throws IOException {
+ if (closed.get()) {
+ throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+ }
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ super.close();
+ if (closed.compareAndSet(false, true)) {
+ closeStream(false);
+ }
+ }
+
+ // for test
+ public long nextExpectPos() {
+ return currPos;
+ }
+
+ @Override
+ public synchronized int available() throws IOException {
+ checkNotClosed();
+ return Ints.saturatedCast(contentLength - nextPos);
+ }
+
+ @VisibleForTesting
+ ObjectRangeInputStream stream() {
+ return stream;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectOutputStream.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectOutputStream.java
new file mode 100644
index 00000000000..aa2bffb6998
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectOutputStream.java
@@ -0,0 +1,336 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.object.staging.FileStagingPart;
+import org.apache.hadoop.fs.tosfs.object.staging.StagingPart;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.SequenceInputStream;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+public class ObjectOutputStream extends OutputStream {
+ private static final Logger LOG = LoggerFactory.getLogger(ObjectOutputStream.class);
+
+ private final ObjectStorage storage;
+ private final ExecutorService uploadPool;
+ private long totalWroteSize;
+ private final String destKey;
+ private final String destScheme;
+ private final long multiUploadThreshold;
+ private final long byteSizePerPart;
+ private final int stagingBufferSize;
+ private final boolean allowPut;
+ private final List<File> stagingDirs;
+ private final List<StagingPart> stagingParts = Lists.newArrayList();
+
+ // For multipart uploads.
+ private final AtomicInteger partNumGetter = new AtomicInteger(0);
+ private MultipartUpload multipartUpload = null;
+ private final List<CompletableFuture<Part>> results = Lists.newArrayList();
+
+ private StagingPart curPart;
+ private final AtomicBoolean closed = new AtomicBoolean(false);
+
+ public ObjectOutputStream(ObjectStorage storage, ExecutorService threadPool, Configuration conf,
+ Path dest, boolean allowPut) {
+ this.storage = storage;
+ this.uploadPool = threadPool;
+ this.destScheme = dest.toUri().getScheme();
+ this.totalWroteSize = 0;
+ this.destKey = createDestKey(dest);
+ this.multiUploadThreshold = conf.getLong(ConfKeys.FS_MULTIPART_THRESHOLD.key(destScheme),
+ ConfKeys.FS_MULTIPART_THRESHOLD_DEFAULT);
+ this.byteSizePerPart =
+ conf.getLong(ConfKeys.FS_MULTIPART_SIZE.key(destScheme), ConfKeys.FS_MULTIPART_SIZE_DEFAULT);
+ this.stagingBufferSize = conf.getInt(ConfKeys.FS_MULTIPART_STAGING_BUFFER_SIZE.key(destScheme),
+ ConfKeys.FS_MULTIPART_STAGING_BUFFER_SIZE_DEFAULT);
+ this.allowPut = allowPut;
+ this.stagingDirs = createStagingDirs(conf, destScheme);
+
+ if (!allowPut) {
+ this.multipartUpload = storage.createMultipartUpload(destKey);
+ }
+ }
+
+ private static List<File> createStagingDirs(Configuration conf, String scheme) {
+ String[] dirs = conf.getStrings(ConfKeys.FS_MULTIPART_STAGING_DIR.key(scheme),
+ ConfKeys.FS_MULTIPART_STAGING_DIR_DEFAULT);
+ Preconditions.checkArgument(dirs != null && dirs.length > 0, "'%s' cannot be an empty list",
+ ConfKeys.FS_MULTIPART_STAGING_DIR.key(scheme));
+
+ List<File> stagingDirs = new ArrayList<>();
+ for (String dir : dirs) {
+ // Create the directory if not exist.
+ File stagingDir = new File(dir);
+ if (!stagingDir.exists() && stagingDir.mkdirs()) {
+ Preconditions.checkArgument(stagingDir.setWritable(true, false),
+ "Failed to change staging dir permission to writable, please check %s with value %s",
+ ConfKeys.FS_MULTIPART_STAGING_DIR.key(scheme), dir);
+ Preconditions.checkArgument(stagingDir.setReadable(true, false),
+ "Failed to change staging dir permission to readable, please check %s with value %s",
+ ConfKeys.FS_MULTIPART_STAGING_DIR.key(scheme), dir);
+ } else {
+ Preconditions.checkArgument(stagingDir.exists(),
+ "Failed to create staging dir, please check %s with value %s",
+ ConfKeys.FS_MULTIPART_STAGING_DIR.key(scheme), dir);
+ Preconditions.checkArgument(stagingDir.isDirectory(),
+ "Staging dir should be a directory, please check %s with value %s",
+ ConfKeys.FS_MULTIPART_STAGING_DIR.key(scheme), dir);
+ }
+ stagingDirs.add(stagingDir);
+ }
+ return stagingDirs;
+ }
+
+ private File chooseStagingDir() {
+ // Choose a random directory from the staging dirs as the candidate staging dir.
+ return stagingDirs.get(ThreadLocalRandom.current().nextInt(stagingDirs.size()));
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ write(new byte[]{(byte) b}, 0, 1);
+ }
+
+ protected String createDestKey(Path dest) {
+ return ObjectUtils.pathToKey(dest);
+ }
+
+ @Override
+ public synchronized void write(byte[] buf, int off, int len) throws IOException {
+ if (len == 0) {
+ return;
+ }
+ Preconditions.checkArgument(off >= 0 && off < buf.length,
+ "Invalid offset - off: %s, len: %s, bufferSize: %s", off, len, buf.length);
+ Preconditions.checkArgument(len >= 0 && off + len <= buf.length,
+ "Invalid length - off: %s, len: %s, bufferSize: %s", off, len, buf.length);
+ Preconditions.checkState(!closed.get(), "OutputStream is closed.");
+
+ while (len > 0) {
+ if (curPart == null) {
+ curPart = newStagingPart();
+ }
+
+ Preconditions.checkArgument(curPart.size() <= byteSizePerPart,
+ "Invalid staging size (%s) which is greater than part size (%s)", curPart.size(), byteSizePerPart);
+
+ // size is the remaining length to fill a complete upload part.
+ int size = (int) Math.min(byteSizePerPart - curPart.size(), len);
+ curPart.write(buf, off, size);
+
+ off += size;
+ len -= size;
+ totalWroteSize += size;
+
+ // Switch to the next staging part if current staging part is full.
+ if (curPart.size() >= byteSizePerPart) {
+ curPart.complete();
+
+ // Upload this part if multipart upload was triggered.
+ if (multipartUpload != null) {
+ CompletableFuture<Part> result = asyncUploadPart(curPart, partNumGetter.incrementAndGet());
+ results.add(result);
+ }
+
+ // Reset the stagingOut
+ curPart = null;
+ }
+
+ // Trigger the multipart upload when reach the configured threshold.
+ if (multipartUpload == null && totalWroteSize >= multiUploadThreshold) {
+ multipartUpload = storage.createMultipartUpload(destKey);
+ Preconditions.checkState(byteSizePerPart >= multipartUpload.minPartSize(),
+ "Configured upload part size %s must be greater than or equals to the minimal part size %s,"
+ + " please check configure key %s.", byteSizePerPart, multipartUpload.minPartSize(),
+ ConfKeys.FS_MULTIPART_THRESHOLD.key(destScheme));
+
+ // Upload the accumulated staging files whose length >= byteSizePerPart.
+ for (StagingPart stagingPart : stagingParts) {
+ if (stagingPart.size() >= byteSizePerPart) {
+ CompletableFuture<Part> result = asyncUploadPart(stagingPart, partNumGetter.incrementAndGet());
+ results.add(result);
+ }
+ }
+ }
+ }
+ }
+
+ private CompletableFuture<Part> asyncUploadPart(final StagingPart stagingPart, final int partNum) {
+ final MultipartUpload immutableUpload = multipartUpload;
+ return CompletableFuture.supplyAsync(() -> uploadPart(stagingPart, partNum), uploadPool)
+ .whenComplete((part, err) -> {
+ stagingPart.cleanup();
+ if (err != null) {
+ LOG.error("Failed to upload part, multipartUpload: {}, partNum: {}, stagingPart: {}",
+ immutableUpload, partNum, stagingPart, err);
+ }
+ });
+ }
+
+ private CompletableFuture<Part> asyncUploadEmptyPart(final int partNum) {
+ final MultipartUpload immutableUpload = multipartUpload;
+ return CompletableFuture.supplyAsync(
+ () -> storage.uploadPart(
+ immutableUpload.key(),
+ immutableUpload.uploadId(),
+ partNum,
+ () -> new ByteArrayInputStream(new byte[0]),
+ 0),
+ uploadPool)
+ .whenComplete((part, err) -> {
+ if (err != null) {
+ LOG.error("Failed to upload empty part, multipartUpload: {}, partNum: {}",
+ immutableUpload, partNum, err);
+ }
+ });
+ }
+
+ private Part uploadPart(StagingPart stagingPart, int partNum) {
+ Preconditions.checkNotNull(storage, "Object storage cannot be null.");
+ Preconditions.checkNotNull(multipartUpload, "Multipart upload is not initialized.");
+ return storage.uploadPart(multipartUpload.key(), multipartUpload.uploadId(),
+ partNum, stagingPart::newIn, stagingPart.size());
+ }
+
+ protected void finishUpload(String key, String uploadId, List<Part> parts) throws IOException {
+ storage.completeUpload(key, uploadId, parts);
+ }
+
+ private void simplePut() throws IOException {
+ if (curPart != null) {
+ curPart.complete();
+ }
+ storage.put(
+ destKey,
+ () -> stagingParts()
+ .stream()
+ .map(StagingPart::newIn)
+ .reduce(SequenceInputStream::new)
+ .orElseGet(() -> new ByteArrayInputStream(new byte[0])),
+ stagingParts().stream().mapToLong(StagingPart::size).sum());
+ // Reset the staging output stream.
+ curPart = null;
+ }
+
+ synchronized List<Part> waitForPartsUpload() {
+ Preconditions.checkArgument(multipartUpload != null, "Multipart upload cannot be null");
+ Preconditions.checkArgument(!results.isEmpty(), "Upload parts cannot be empty");
+ // Waiting for all the upload parts to be finished.
+ return results.stream()
+ .map(CompletableFuture::join)
+ .sorted(Comparator.comparing(Part::num))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (!closed.compareAndSet(false, true)) {
+ return;
+ }
+
+ try {
+ // Use the simple PUT API if wrote bytes is not reached the multipart threshold.
+ if (multipartUpload == null && allowPut) {
+ simplePut();
+ return;
+ }
+ Preconditions.checkNotNull(multipartUpload, "MultipartUpload cannot be null since allowPut was disabled.");
+
+ // Use multipart upload API to upload those parts.
+ if (totalWroteSize <= 0) {
+ // Write an empty part for this zero-byte file.
+ CompletableFuture<Part> result = asyncUploadEmptyPart(partNumGetter.incrementAndGet());
+ results.add(result);
+ } else if (curPart != null) {
+ curPart.complete();
+ // Submit the last part to upload thread pool.
+ CompletableFuture<Part> result = asyncUploadPart(curPart, partNumGetter.incrementAndGet());
+ results.add(result);
+ // Reset the staging output stream.
+ curPart = null;
+ }
+
+ // Finish the multipart uploads.
+ finishUpload(multipartUpload.key(), multipartUpload.uploadId(), waitForPartsUpload());
+
+ } catch (Exception e) {
+ LOG.error("Encountering error when closing output stream", e);
+ if (multipartUpload != null) {
+ CommonUtils.runQuietly(() -> storage.abortMultipartUpload(multipartUpload.key(), multipartUpload.uploadId()));
+ }
+ throw e;
+ } finally {
+ // Clear all the staging part.
+ deleteStagingPart(stagingParts);
+ }
+ }
+
+ public long totalWroteSize() {
+ return totalWroteSize;
+ }
+
+ public ObjectStorage storage() {
+ return storage;
+ }
+
+ public List<StagingPart> stagingParts() {
+ return stagingParts;
+ }
+
+ public String destKey() {
+ return destKey;
+ }
+
+ public MultipartUpload upload() {
+ return multipartUpload;
+ }
+
+ private void deleteStagingPart(List<StagingPart> parts) {
+ for (StagingPart part : parts) {
+ part.cleanup();
+ }
+ }
+
+ private StagingPart newStagingPart() {
+ String stagingPath = String.format("%s/staging-%s.tmp", chooseStagingDir(), UUIDUtils.random());
+ StagingPart part = new FileStagingPart(stagingPath, stagingBufferSize);
+ stagingParts.add(part);
+ return part;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectRangeInputStream.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectRangeInputStream.java
new file mode 100644
index 00000000000..77d180323ff
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectRangeInputStream.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.util.FSUtils;
+import org.apache.hadoop.fs.tosfs.util.Range;
+import org.apache.hadoop.thirdparty.com.google.common.io.ByteStreams;
+import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
+import org.apache.hadoop.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class ObjectRangeInputStream extends FSInputStream {
+ private static final Logger LOG = LoggerFactory.getLogger(ObjectRangeInputStream.class);
+ private static final int MAX_SKIP_SIZE = 1024 * 1024;
+
+ private final ObjectStorage storage;
+ private final String objectKey;
+ private final Range range;
+ private final byte[] checksum;
+
+ private InputStream stream;
+ private long nextPos;
+ private long currPos;
+ private boolean closed = false;
+
+ public ObjectRangeInputStream(ObjectStorage storage, Path path, Range range, byte[] checksum) {
+ this(storage, ObjectUtils.pathToKey(path), range, checksum);
+ }
+
+ public ObjectRangeInputStream(
+ ObjectStorage storage, String objectKey, Range range, byte[] checksum) {
+ this.storage = storage;
+ this.objectKey = objectKey;
+ this.range = range;
+ this.checksum = checksum;
+
+ this.stream = null;
+ this.nextPos = range.off();
+ this.currPos = nextPos;
+
+ Preconditions.checkNotNull(checksum, "Checksum should not be null.");
+ }
+
+ @Override
+ public int read() throws IOException {
+ byte[] buf = new byte[1];
+ int n = read(buf, 0, buf.length);
+ if (n < 0) {
+ return -1;
+ } else {
+ return buf[0] & 0xFF;
+ }
+ }
+
+ @Override
+ public int read(byte[] buffer, int offset, int length) throws IOException {
+ checkNotClosed();
+ FSUtils.checkReadParameters(buffer, offset, length);
+
+ if (length == 0) {
+ return 0;
+ }
+
+ if (!range.include(nextPos)) {
+ return -1;
+ }
+
+ seekStream();
+
+ int toRead = Math.min(length, Ints.saturatedCast(range.end() - nextPos));
+ int readLen = stream.read(buffer, offset, toRead);
+ if (readLen > 0) {
+ nextPos += readLen;
+ currPos += readLen;
+ }
+ return readLen;
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ closeStream();
+ closed = true;
+ }
+
+ @Override
+ public int read(long position, byte[] buffer, int offset, int length) throws IOException {
+ checkNotClosed();
+
+ FSUtils.checkReadParameters(buffer, offset, length);
+ if (!range.include(position)) {
+ return -1;
+ }
+
+ int toRead = Math.min(length, Ints.saturatedCast(range.end() - position));
+ if (toRead == 0) {
+ return 0;
+ }
+
+ try (InputStream in = openStream(position, toRead)) {
+ return in.read(buffer, offset, toRead);
+ }
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ checkNotClosed();
+ Preconditions.checkArgument(range.include(pos), "Position %s must be in range %s", pos, range);
+ this.nextPos = pos;
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ checkNotClosed();
+ return nextPos;
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ checkNotClosed();
+ return false;
+ }
+
+ private void seekStream() throws IOException {
+ // sequential read
+ if (stream != null && nextPos == currPos) {
+ return;
+ }
+
+ // random read
+ if (stream != null && nextPos > currPos) {
+ long skip = nextPos - currPos;
+ // It is not worth skipping because the skip size is too big, or it can't read any bytes after skip.
+ if (skip < MAX_SKIP_SIZE) {
+ try {
+ ByteStreams.skipFully(stream, skip);
+ currPos = nextPos;
+ return;
+ } catch (IOException ignored) {
+ LOG.warn("Failed to skip {} bytes in stream, will try to reopen the stream", skip);
+ }
+ }
+ }
+
+ currPos = nextPos;
+
+ closeStream();
+ stream = openStream(nextPos, range.end() - nextPos);
+ }
+
+ private InputStream openStream(long offset, long limit) throws IOException {
+ return storage.get(objectKey, offset, limit).verifiedStream(checksum);
+ }
+
+ private void closeStream() throws IOException {
+ if (stream != null) {
+ stream.close();
+ }
+ stream = null;
+ }
+
+ private void checkNotClosed() throws IOException {
+ if (closed) {
+ throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+ }
+ }
+
+ public boolean include(long pos) {
+ return range.include(pos);
+ }
+
+ public Range range() {
+ return range;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectStorage.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectStorage.java
new file mode 100644
index 00000000000..6c2f5d4f98a
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectStorage.java
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.object.exceptions.InvalidObjectKeyException;
+import org.apache.hadoop.fs.tosfs.util.LazyReload;
+import org.apache.hadoop.fs.tosfs.object.exceptions.NotAppendableException;
+import org.apache.hadoop.fs.tosfs.object.request.ListObjectsRequest;
+import org.apache.hadoop.fs.tosfs.object.response.ListObjectsResponse;
+
+import java.io.ByteArrayInputStream;
+import java.io.Closeable;
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+public interface ObjectStorage extends Closeable {
+ String EMPTY_DELIMITER = "";
+
+ /**
+ * @return Scheme of the object storage.
+ */
+ String scheme();
+
+ /**
+ * @return null if bucket doesn't exist.
+ */
+ BucketInfo bucket();
+
+ /**
+ * Initialize the Object storage, according to the properties.
+ *
+ * @param conf to initialize the {@link ObjectStorage}
+ * @param bucket the corresponding bucket name, each object store has one bucket.
+ */
+ void initialize(Configuration conf, String bucket);
+
+ /**
+ * @return storage conf
+ */
+ Configuration conf();
+
+ default ObjectContent get(String key) {
+ return get(key, 0, -1);
+ }
+
+ /**
+ * Get the data for the given object specified by key.
+ * Throw {@link RuntimeException} if object key doesn't exist.
+ * Throw {@link RuntimeException} if object key is null or empty.
+ *
+ * @return {@link InputStream} to read the object content.
+ */
+ ObjectContent get(String key, long offset, long limit);
+
+ default byte[] put(String key, byte[] data) {
+ return put(key, data, 0, data.length);
+ }
+
+ default byte[] put(String key, byte[] data, int off, int len) {
+ return put(key, () -> new ByteArrayInputStream(data, off, len), len);
+ }
+
+ /**
+ * Put data read from a reader to an object specified by key. The implementation must ensure to
+ * close the stream created by stream provider after finishing stream operation.
+ * Throw {@link RuntimeException} if object key is null or empty.
+ *
+ * @param key for the object.
+ * @param streamProvider the binary input stream provider that create input stream to write.
+ * @param contentLength the content length, if the actual data is bigger than content length, the
+ * object can be created, but the object data will be truncated to the given
+ * content length, if the actual data is smaller than content length, will
+ * create object failed with unexpect end of IOException.
+ * @return the checksum of uploaded object
+ */
+ byte[] put(String key, InputStreamProvider streamProvider, long contentLength);
+
+ default byte[] append(String key, byte[] data) {
+ return append(key, data, 0, data.length);
+ }
+
+ default byte[] append(String key, byte[] data, int off, int len) {
+ return append(key, () -> new ByteArrayInputStream(data, off, len), len);
+ }
+
+ /**
+ * Append data read from a reader to an object specified by key. If the object exists, data will
+ * be appended to the tail. Otherwise, the object will be created and data will be written to it.
+ * Content length could be zero if object exists. If the object doesn't exist and content length
+ * is zero, a {@link NotAppendableException} will be thrown.
+ * <p>
+ * The first one wins if there are concurrent appends.
+ * <p>
+ * The implementation must ensure to close the stream created by stream provider after finishing
+ * stream operation.
+ * Throw {@link RuntimeException} if object key is null or empty.
+ *
+ * @param key for the object.
+ * @param streamProvider the binary input stream provider that create input stream to write.
+ * @param contentLength the appended content length. If the actual appended data is bigger than
+ * content length, the object can be appended but the data to append will be
+ * truncated to the given content length. If the actual data is smaller than
+ * content length, append object will fail with unexpect end IOException.
+ * @return the checksum of appended object.
+ * @throws NotAppendableException if the object already exists and is not appendable, or the
+ * object doesn't exist and content length is zero.
+ */
+ byte[] append(String key, InputStreamProvider streamProvider, long contentLength);
+
+ /**
+ * Delete an object.
+ * No exception thrown if the object key doesn't exist.
+ * Throw {@link RuntimeException} if object key is null or empty.
+ *
+ * @param key the given object key to be deleted.
+ */
+ void delete(String key);
+
+ /**
+ * Delete multiple keys. If one key doesn't exist, it will be treated as delete succeed, won't be
+ * included in response list.
+ *
+ * @param keys the given object keys to be deleted
+ * @return the keys delete failed
+ */
+ List<String> batchDelete(List<String> keys);
+
+ /**
+ * Delete all objects with the given prefix(include the prefix if the corresponding object exists)
+ *
+ * @param prefix the prefix key.
+ */
+ void deleteAll(String prefix);
+
+ /**
+ * Head returns some information about the object or a null if not found.
+ * Throw {@link RuntimeException} if object key is null or empty.
+ * There are some differences between directory bucket and general purpose bucket:
+ * <ul>
+ * <li>Assume an file object 'a/b' exists, only head("a/b") will get the meta of object 'a/b'
+ * for both general purpose bucket and directory bucket</li>
+ * <li>Assume an dir object 'a/b/' exists, regarding general purpose bucket, only head("a/b/")
+ * will get the meta of object 'a/b/', but for directory bucket, both head("a/b") and
+ * head("a/b/") will get the meta of object 'a/b/'</li>
+ * </ul>
+ *
+ * @param key for the specified object.
+ * @return {@link ObjectInfo}, null if the object does not exist.
+ * @throws InvalidObjectKeyException if the object is locating under an existing file in directory
+ * bucket, which is not allowed.
+ */
+ ObjectInfo head(String key);
+
+ /**
+ * List objects according to the given {@link ListObjectsRequest}
+ *
+ * @param request {@link ListObjectsRequest}
+ * @return the iterable of {@link ListObjectsResponse} which contains objects and common prefixes
+ */
+ Iterable<ListObjectsResponse> list(ListObjectsRequest request);
+
+ /**
+ * List limited objects in a given bucket
+ *
+ * @param prefix Limits the response to keys that begin with the specified prefix.
+ * @param startAfter StartAfter is where you want the object storage to start listing from.
+ * object storage starts listing after this specified key.
+ * StartAfter can be any key in the bucket.
+ * @param limit Limit the maximum number of response objects.
+ * @return {@link ObjectInfo} the object list with matched prefix key
+ */
+ default Iterable<ObjectInfo> list(String prefix, String startAfter, int limit) {
+ ListObjectsRequest request = ListObjectsRequest.builder()
+ .prefix(prefix)
+ .startAfter(startAfter)
+ .maxKeys(limit)
+ .delimiter(EMPTY_DELIMITER)
+ .build();
+
+ return new LazyReload<>(() -> {
+ Iterator<ListObjectsResponse> iterator = list(request).iterator();
+ return buf -> {
+ if (!iterator.hasNext()) {
+ return true;
+ }
+ buf.addAll(iterator.next().objects());
+
+ return !iterator.hasNext();
+ };
+ });
+ }
+
+ /**
+ * List all objects in a given bucket
+ *
+ * @param prefix Limits the response to keys that begin with the specified prefix.
+ * @param startAfter StartAfter is where you want the object storage to start listing from.
+ * object storage starts listing after this specified key.
+ * StartAfter can be any key in the bucket.
+ * @return {@link ObjectInfo} Iterable to iterate over the objects with matched prefix key
+ * and StartAfter
+ */
+ default Iterable<ObjectInfo> listAll(String prefix, String startAfter) {
+ return list(prefix, startAfter, -1);
+ }
+
+ /**
+ * CreateMultipartUpload starts to upload a large object part by part.
+ *
+ * @param key for the specified object.
+ * @return {@link MultipartUpload}.
+ */
+ MultipartUpload createMultipartUpload(String key);
+
+ /**
+ * UploadPart upload a part of an object. The implementation must ensure to close the stream
+ * created by stream provider after finishing stream operation.
+ *
+ * @param key for the specified object.
+ * @param uploadId for the multipart upload id.
+ * @param partNum upload part number.
+ * @param streamProvider the stream provider to provider part stream
+ * @param contentLength the content length, if the actual data is bigger than content length, the
+ * object can be created, but the object data will be truncated to the given
+ * content length, if the actual data is smaller than content length, will
+ * create object failed with unexpect end of IOException.
+ */
+ Part uploadPart(String key, String uploadId, int partNum, InputStreamProvider streamProvider,
+ long contentLength);
+
+ /**
+ * Complete the multipart uploads with given object key and upload id.
+ *
+ * @param key for the specified object.
+ * @param uploadId id of the multipart upload.
+ * @param uploadParts parts to upload.
+ * @return the checksum of uploaded object
+ */
+ byte[] completeUpload(String key, String uploadId, List<Part> uploadParts);
+
+ /**
+ * Abort a multipart upload.
+ *
+ * @param key object key.
+ * @param uploadId multipart upload Id.
+ */
+ void abortMultipartUpload(String key, String uploadId);
+
+ /**
+ * List multipart uploads under a path.
+ *
+ * @param prefix for uploads to abort.
+ * @return Iterable to iterate over multipart unloads.
+ */
+ Iterable<MultipartUpload> listUploads(String prefix);
+
+ /**
+ * upload part copy with mutipart upload id
+ *
+ * @param srcKey source object key
+ * @param dstKey dest object key
+ * @param uploadId id of the multipart upload copy
+ * @param partNum part num of the multipart upload copy
+ * @param copySourceRangeStart copy source range start of source object
+ * @param copySourceRangeEnd copy source range end of source object
+ * @return {@link Part}.
+ */
+ Part uploadPartCopy(
+ String srcKey, String dstKey, String uploadId, int partNum, long copySourceRangeStart,
+ long copySourceRangeEnd);
+
+ /**
+ * Copy binary content from one object to another object.
+ *
+ * @param srcKey source object key
+ * @param dstKey dest object key
+ */
+ void copy(String srcKey, String dstKey);
+
+ /**
+ * Atomic rename source object to dest object without any data copying.
+ * Will overwrite dest object if dest object exists.
+ *
+ * @param srcKey source object key
+ * @param dstKey dest object key
+ * @throws RuntimeException if rename failed，e.g. srcKey is equal to dstKey or the source object
+ * doesn't exist.
+ */
+ void rename(String srcKey, String dstKey);
+
+ /**
+ * Attach tags to specified object. This method will overwrite all existed tags with the new tags.
+ * Remove all existed tags if the new tags are empty. The maximum tags number is 10.
+ *
+ * @param key the key of the object key.
+ * @param newTags the new tags to put.
+ * @throws RuntimeException if key doesn't exist.
+ */
+ default void putTags(String key, Map<String, String> newTags) {
+ throw new UnsupportedOperationException(
+ this.getClass().getSimpleName() + " doesn't support putObjectTagging.");
+ }
+
+ /**
+ * Get all attached tags of the object.
+ *
+ * @param key the key of the object.
+ * @return map containing all tags.
+ * @throws RuntimeException if key doesn't exist.
+ */
+ default Map<String, String> getTags(String key) {
+ throw new UnsupportedOperationException(
+ this.getClass().getSimpleName() + " doesn't support getObjectTagging.");
+ }
+
+ /**
+ * Gets the object status for the given key.
+ * It's different from {@link ObjectStorage#head(String)}, it returns object info if the key
+ * exists or the prefix with value key exists.
+ * <p>
+ * There are three kinds of implementations:
+ * <ul>
+ * <li>Uses the headObject API if the object storage support directory bucket and the requested
+ * bucket is a directory bucket, the object storage will return object directly if the file or
+ * dir exists, otherwise return null</li>
+ * <li>Uses getFileStatus API if the object storage support it, e.g. TOS. The object storage
+ * will return the object directly if the key or prefix exists, otherwise return null.</li>
+ * <li>If the object storage doesn't support above all cases, you have to try to headObject(key)
+ * at first, if the object doesn't exist, and then headObject(key + "/") later if the key
+ * doesn't end with '/', and if neither the new key doesn't exist, and then use listObjects API
+ * to check whether the prefix/key exist.</li>
+ * </ul>
+ *
+ * @param key the object
+ * @return object info if the key or prefix exists, otherwise return null.
+ * @throws InvalidObjectKeyException if the object is locating under an existing file in directory
+ * bucket, which is not allowed.
+ */
+ ObjectInfo objectStatus(String key);
+
+ /**
+ * Get the object storage checksum information, including checksum algorithm name,
+ * checksum type, etc.
+ *
+ * @return checksum information of this storage.
+ */
+ ChecksumInfo checksumInfo();
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectStorageFactory.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectStorageFactory.java
new file mode 100644
index 00000000000..1f86ecf456b
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectStorageFactory.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.object.tos.TOS;
+import org.apache.hadoop.util.Preconditions;
+
+import java.lang.reflect.InvocationTargetException;
+
+import static org.apache.hadoop.fs.tosfs.conf.ConfKeys.FS_OBJECT_STORAGE_IMPL;
+
+public class ObjectStorageFactory {
+
+ private static final Configuration DEFAULT_IMPLS = new Configuration();
+
+ static {
+ // Setup default object storage impl for scheme "tos" and "filestore".
+ DEFAULT_IMPLS.set(ConfKeys.FS_OBJECT_STORAGE_IMPL.key("tos"), TOS.class.getName());
+ DEFAULT_IMPLS.set(ConfKeys.FS_OBJECT_STORAGE_IMPL.key("filestore"), FileStore.class.getName());
+ }
+
+ private ObjectStorageFactory() {
+ }
+
+ public static ObjectStorage createWithPrefix(String prefix, String scheme, String bucket, Configuration conf) {
+ ObjectStorage storage = create(scheme, bucket, conf);
+ return new PrefixStorage(storage, prefix);
+ }
+
+ public static ObjectStorage create(String scheme, String bucket, Configuration conf) {
+ Preconditions.checkArgument(StringUtils.isNotEmpty(scheme), "Scheme is null or empty.");
+ Preconditions.checkArgument(StringUtils.isNotEmpty(bucket), "Bucket is null or empty.");
+ Preconditions.checkNotNull(conf, "Conf is null.");
+
+ try {
+ String confKey = FS_OBJECT_STORAGE_IMPL.key(scheme);
+ String impl = conf.get(confKey, DEFAULT_IMPLS.get(confKey));
+
+ Preconditions.checkArgument(StringUtils.isNotEmpty(impl),
+ "Cannot locate the ObjectStorage implementation for scheme '%s'", scheme);
+ ObjectStorage store = (ObjectStorage) Class.forName(impl).getDeclaredConstructor().newInstance();
+ store.initialize(conf, bucket);
+ return store;
+ } catch (ClassNotFoundException |
+ InvocationTargetException |
+ InstantiationException |
+ IllegalAccessException |
+ NoSuchMethodException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectUtils.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectUtils.java
new file mode 100644
index 00000000000..94f4bee1f8d
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectUtils.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.util.Range;
+import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
+
+import java.util.List;
+
+public class ObjectUtils {
+ public static final String SLASH = "/";
+
+ private ObjectUtils() {
+ }
+
+ public static Path keyToPath(String key) {
+ return new Path(SLASH + key);
+ }
+
+ public static String path(String key) {
+ return key.startsWith(SLASH) ? key : SLASH + key;
+ }
+
+ public static String pathToKey(Path p) {
+ return pathToKey(p, false);
+ }
+
+ public static String pathToKey(Path p, Boolean isDir) {
+ Preconditions.checkArgument(p != null, "Null path");
+ if (p.toUri().getScheme() != null && p.toUri().getPath().isEmpty()) {
+ return "";
+ }
+ String key = p.toUri().getPath().substring(1);
+ if (isDir && !key.isEmpty()) {
+ return key.endsWith(SLASH) ? key : key + SLASH;
+ }
+ return key;
+ }
+
+ public static void deleteAllObjects(ObjectStorage storage, Iterable<ObjectInfo> objects, int batchSize) {
+ List<String> keysToDelete = Lists.newArrayList();
+ for (ObjectInfo obj : objects) {
+ keysToDelete.add(obj.key());
+
+ if (keysToDelete.size() == batchSize) {
+ batchDelete(storage, keysToDelete);
+ keysToDelete.clear();
+ }
+ }
+
+ if (!keysToDelete.isEmpty()) {
+ batchDelete(storage, keysToDelete);
+ }
+ }
+
+ private static void batchDelete(ObjectStorage storage, List<String> keys) {
+ List<String> failedKeys = storage.batchDelete(keys);
+ if (!failedKeys.isEmpty()) {
+ throw new RuntimeException(String.format("Failed to delete %s objects, detail: %s",
+ failedKeys.size(), Joiner.on(",").join(failedKeys)));
+ }
+ }
+
+ public static Range calculateRange(final long offset, final long limit, final long objSize) {
+ Preconditions.checkArgument(offset >= 0, String.format("offset is a negative number: %s", offset));
+ Preconditions.checkArgument(offset <= objSize,
+ String.format("offset: %s is bigger than object size: %s", offset, objSize));
+ long len = limit < 0 ? objSize - offset : Math.min(objSize - offset, limit);
+ return Range.of(offset, len);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/Part.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/Part.java
new file mode 100644
index 00000000000..ace7acaf28a
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/Part.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
+
+import java.util.Objects;
+
+public class Part {
+ private int num;
+ private long size;
+ private String eTag;
+
+ // No-arg constructor for json serializer, don't use.
+ public Part() {
+ }
+
+ public Part(int num, long size, String eTag) {
+ this.num = num;
+ this.size = size;
+ this.eTag = eTag;
+ }
+
+ public int num() {
+ return num;
+ }
+
+ public long size() {
+ return size;
+ }
+
+ public String eTag() {
+ return eTag;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(num, size, eTag);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ } else if (!(o instanceof Part)) {
+ return false;
+ }
+ Part that = (Part) o;
+ return Objects.equals(num, that.num)
+ && Objects.equals(size, that.size)
+ && Objects.equals(eTag, that.eTag);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("PartNum", num)
+ .add("PartSize", size)
+ .add("ETag", eTag)
+ .toString();
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/PrefixStorage.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/PrefixStorage.java
new file mode 100644
index 00000000000..84408dc972d
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/PrefixStorage.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.object.request.ListObjectsRequest;
+import org.apache.hadoop.fs.tosfs.object.response.ListObjectsResponse;
+import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
+import org.apache.hadoop.util.Preconditions;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class PrefixStorage implements DirectoryStorage {
+ private final ObjectStorage storage;
+ private final String prefix;
+
+ public PrefixStorage(ObjectStorage storage, String prefix) {
+ this.storage = storage;
+ this.prefix = prefix;
+ }
+
+ @VisibleForTesting
+ ObjectStorage storage() {
+ return storage;
+ }
+
+ @VisibleForTesting
+ String prefix() {
+ return prefix;
+ }
+
+ @Override
+ public String scheme() {
+ return storage.scheme();
+ }
+
+ @Override
+ public BucketInfo bucket() {
+ return storage.bucket();
+ }
+
+ @Override
+ public void initialize(Configuration conf, String bucket) {
+ storage.initialize(conf, bucket);
+ }
+
+ @Override
+ public Configuration conf() {
+ return storage.conf();
+ }
+
+ @Override
+ public ObjectContent get(String key, long offset, long limit) {
+ Preconditions.checkArgument(key != null && key.length() > 0,
+ "Object key cannot be null or empty.");
+ return storage.get(prefix + key, offset, limit);
+ }
+
+ @Override
+ public byte[] put(String key, InputStreamProvider streamProvider, long contentLength) {
+ Preconditions.checkArgument(key != null && key.length() > 0,
+ "Object key cannot be null or empty.");
+ return storage.put(prefix + key, streamProvider, contentLength);
+ }
+
+ @Override
+ public byte[] append(String key, InputStreamProvider streamProvider, long contentLength) {
+ Preconditions.checkArgument(key != null && key.length() > 0,
+ "Object key cannot be null or empty.");
+ return storage.append(prefix + key, streamProvider, contentLength);
+ }
+
+ @Override
+ public void delete(String key) {
+ Preconditions.checkArgument(key != null, "Object key cannot be null or empty.");
+ storage.delete(prefix + key);
+ }
+
+ @Override
+ public List<String> batchDelete(List<String> keys) {
+ return storage.batchDelete(keys.stream().map(key -> prefix + key).collect(Collectors.toList()));
+ }
+
+ @Override
+ public void deleteAll(String prefix) {
+ storage.deleteAll(this.prefix + prefix);
+ }
+
+ @Override
+ public ObjectInfo head(String key) {
+ Preconditions.checkArgument(key != null && key.length() > 0,
+ "Object key cannot be null or empty.");
+ return removePrefix(storage.head(prefix + key));
+ }
+
+ private ListObjectsResponse removePrefix(ListObjectsResponse response) {
+ List<ObjectInfo> objects = response.objects().stream()
+ .map(this::removePrefix)
+ .collect(Collectors.toList());
+ List<String> commonPrefixKeys = response.commonPrefixes().stream()
+ .map(this::removePrefix)
+ .collect(Collectors.toList());
+ return new ListObjectsResponse(objects, commonPrefixKeys);
+ }
+
+ @Override
+ public Iterable<ListObjectsResponse> list(ListObjectsRequest request) {
+ String startAfter = Strings.isNullOrEmpty(request.startAfter()) ?
+ request.startAfter() : prefix + request.startAfter();
+
+ ListObjectsRequest newReq = ListObjectsRequest.builder()
+ .prefix(prefix + request.prefix())
+ .startAfter(startAfter)
+ .maxKeys(request.maxKeys())
+ .delimiter(request.delimiter())
+ .build();
+
+ return Iterables.transform(storage.list(newReq), this::removePrefix);
+ }
+
+ @Override
+ public MultipartUpload createMultipartUpload(String key) {
+ Preconditions.checkArgument(key != null && key.length() > 0,
+ "Object key cannot be null or empty.");
+ return removePrefix(storage.createMultipartUpload(prefix + key));
+ }
+
+ @Override
+ public Part uploadPart(
+ String key, String uploadId, int partNum,
+ InputStreamProvider streamProvider, long contentLength) {
+ Preconditions.checkArgument(key != null && key.length() > 0,
+ "Object key cannot be null or empty.");
+ return storage.uploadPart(prefix + key, uploadId, partNum, streamProvider, contentLength);
+ }
+
+ @Override
+ public byte[] completeUpload(String key, String uploadId, List<Part> uploadParts) {
+ Preconditions.checkArgument(key != null && key.length() > 0,
+ "Object key cannot be null or empty.");
+ return storage.completeUpload(prefix + key, uploadId, uploadParts);
+ }
+
+ @Override
+ public void abortMultipartUpload(String key, String uploadId) {
+ Preconditions.checkArgument(key != null && key.length() > 0,
+ "Object key cannot be null or empty.");
+ storage.abortMultipartUpload(prefix + key, uploadId);
+ }
+
+ @Override
+ public Iterable<MultipartUpload> listUploads(String keyPrefix) {
+ return Iterables.transform(storage.listUploads(prefix + keyPrefix), this::removePrefix);
+ }
+
+ @Override
+ public Part uploadPartCopy(
+ String srcKey, String dstKey, String uploadId, int partNum, long copySourceRangeStart,
+ long copySourceRangeEnd) {
+ return storage.uploadPartCopy(prefix + srcKey, prefix + dstKey, uploadId, partNum,
+ copySourceRangeStart, copySourceRangeEnd);
+ }
+
+ @Override
+ public void copy(String srcKey, String dstKey) {
+ storage.copy(prefix + srcKey, prefix + dstKey);
+ }
+
+ @Override
+ public void rename(String srcKey, String dstKey) {
+ storage.rename(prefix + srcKey, prefix + dstKey);
+ }
+
+ private ObjectInfo removePrefix(ObjectInfo o) {
+ if (o == null) {
+ return null;
+ }
+ return new ObjectInfo(removePrefix(o.key()), o.size(), o.mtime(), o.checksum(), o.isDir());
+ }
+
+ private MultipartUpload removePrefix(MultipartUpload u) {
+ if (u == null) {
+ return null;
+ }
+ return new MultipartUpload(removePrefix(u.key()), u.uploadId(), u.minPartSize(),
+ u.maxPartCount());
+ }
+
+ private String removePrefix(String key) {
+ if (key == null) {
+ return null;
+ } else if (key.startsWith(prefix)) {
+ return key.substring(prefix.length());
+ } else {
+ return key;
+ }
+ }
+
+ @Override
+ public void putTags(String key, Map<String, String> newTags) {
+ storage.putTags(prefix + key, newTags);
+ }
+
+ @Override
+ public Map<String, String> getTags(String key) {
+ return storage.getTags(prefix + key);
+ }
+
+ @Override
+ public ObjectInfo objectStatus(String key) {
+ Preconditions.checkArgument(key != null && !key.isEmpty(),
+ "Object key cannot be null or empty.");
+ return removePrefix(storage.objectStatus(prefix + key));
+ }
+
+ @Override
+ public ChecksumInfo checksumInfo() {
+ return storage.checksumInfo();
+ }
+
+ @Override
+ public void close() throws IOException {
+ storage.close();
+ }
+
+ @Override
+ public Iterable<ObjectInfo> listDir(String key, boolean recursive) {
+ Preconditions.checkArgument(storage instanceof DirectoryStorage);
+ return Iterables.transform(((DirectoryStorage) storage).listDir(prefix + key, recursive),
+ this::removePrefix);
+ }
+
+ @Override
+ public void deleteDir(String key, boolean recursive) {
+ Preconditions.checkArgument(storage instanceof DirectoryStorage);
+ ((DirectoryStorage) storage).deleteDir(prefix + key, recursive);
+ }
+
+ @Override
+ public boolean isEmptyDir(String key) {
+ Preconditions.checkArgument(storage instanceof DirectoryStorage);
+ return ((DirectoryStorage) storage).isEmptyDir(prefix + key);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/exceptions/ChecksumMismatchException.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/exceptions/ChecksumMismatchException.java
new file mode 100644
index 00000000000..421d040296f
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/exceptions/ChecksumMismatchException.java
@@ -0,0 +1,16 @@
+package org.apache.hadoop.fs.tosfs.object.exceptions;
+
+import org.apache.hadoop.util.StringUtils;
+
+import java.io.IOException;
+
+public class ChecksumMismatchException extends IOException {
+ public ChecksumMismatchException(String message) {
+ super(message);
+ }
+
+ public ChecksumMismatchException(byte[] expected, byte[] actual) {
+ this(String.format("Expected checksum is %s while actual checksum is %s",
+ StringUtils.byteToHexString(expected), StringUtils.byteToHexString(actual)));
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/exceptions/InvalidObjectKeyException.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/exceptions/InvalidObjectKeyException.java
new file mode 100644
index 00000000000..19897a54443
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/exceptions/InvalidObjectKeyException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.exceptions;
+
+/**
+ * Regarding accessing an object in directory bucket, if the object is locating under an existing file in directory
+ * bucket, the {@link InvalidObjectKeyException} will be thrown. E.g. there is a file object 'a/b/file' exists in
+ * directory bucket, the {@link InvalidObjectKeyException} will be thrown if head object 'a/b/file/c' no matter
+ * whether 'c' exists or not.
+ */
+public class InvalidObjectKeyException extends RuntimeException {
+ public InvalidObjectKeyException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/exceptions/NotAppendableException.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/exceptions/NotAppendableException.java
new file mode 100644
index 00000000000..2a7def11763
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/exceptions/NotAppendableException.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.exceptions;
+
+public class NotAppendableException extends RuntimeException {
+ public NotAppendableException(String msg) {
+ super(msg);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/request/ListObjectsRequest.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/request/ListObjectsRequest.java
new file mode 100644
index 00000000000..18d981930a7
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/request/ListObjectsRequest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.request;
+
+public class ListObjectsRequest {
+ private final String prefix;
+ private final String startAfter;
+ private final int maxKeys;
+ private final String delimiter;
+
+ private ListObjectsRequest(String prefix, String startAfter, int maxKeys, String delimiter) {
+ this.prefix = prefix;
+ this.startAfter = startAfter;
+ this.maxKeys = maxKeys;
+ this.delimiter = delimiter;
+ }
+
+ public String prefix() {
+ return prefix;
+ }
+
+ public String startAfter() {
+ return startAfter;
+ }
+
+ public int maxKeys() {
+ return maxKeys;
+ }
+
+ public String delimiter() {
+ return delimiter;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+ private String prefix;
+ private String startAfter;
+ // -1 means list all object keys
+ private int maxKeys = -1;
+ private String delimiter;
+
+ public Builder prefix(String prefix) {
+ this.prefix = prefix;
+ return this;
+ }
+
+ public Builder startAfter(String startAfter) {
+ this.startAfter = startAfter;
+ return this;
+ }
+
+ public Builder maxKeys(int maxKeys) {
+ this.maxKeys = maxKeys;
+ return this;
+ }
+
+ public Builder delimiter(String delimiter) {
+ this.delimiter = delimiter;
+ return this;
+ }
+
+ public ListObjectsRequest build() {
+ return new ListObjectsRequest(prefix, startAfter, maxKeys, delimiter);
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/response/ListObjectsResponse.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/response/ListObjectsResponse.java
new file mode 100644
index 00000000000..eb3adf8522a
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/response/ListObjectsResponse.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.response;
+
+import org.apache.hadoop.fs.tosfs.object.ObjectInfo;
+
+import java.util.List;
+
+public class ListObjectsResponse {
+ private final List<ObjectInfo> objects;
+ private final List<String> commonPrefixes;
+
+ public ListObjectsResponse(
+ List<ObjectInfo> objects,
+ List<String> commonPrefixes) {
+ this.objects = objects;
+ this.commonPrefixes = commonPrefixes;
+ }
+
+ public List<ObjectInfo> objects() {
+ return objects;
+ }
+
+ public List<String> commonPrefixes() {
+ return commonPrefixes;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/FileStagingPart.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/FileStagingPart.java
new file mode 100644
index 00000000000..3dda0aab904
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/FileStagingPart.java
@@ -0,0 +1,176 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.staging;
+
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
+import org.apache.hadoop.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class FileStagingPart implements StagingPart {
+ private static final Logger LOG = LoggerFactory.getLogger(FileStagingPart.class);
+
+ private final Path path;
+ private final int stagingBufferSize;
+ private final StagingFileOutputStream out;
+ private State state = State.WRITABLE;
+
+ public FileStagingPart(String filePath, int stagingBufferSize) {
+ this.path = Paths.get(filePath);
+ this.stagingBufferSize = stagingBufferSize;
+ this.out = new StagingFileOutputStream(path, stagingBufferSize);
+ }
+
+ @Override
+ public synchronized void write(byte[] b, int off, int len) throws IOException {
+ Preconditions.checkState(state == State.WRITABLE,
+ "Cannot write the part since it's not writable now, state: %s", state);
+ out.write(b, off, len);
+ }
+
+ @Override
+ public synchronized void complete() throws IOException {
+ Preconditions.checkState(state == State.WRITABLE,
+ "Cannot complete the part since it's not writable now, state: %s", state);
+ out.close();
+ state = State.READABLE;
+ }
+
+ @Override
+ public synchronized InputStream newIn() {
+ Preconditions.checkState(state == State.READABLE,
+ "Cannot read the part since it's not readable now, state: %s.", state);
+ return out.newIn();
+ }
+
+ @Override
+ public synchronized long size() {
+ return out.size();
+ }
+
+ @Override
+ public synchronized State state() {
+ return state;
+ }
+
+ @Override
+ public synchronized void cleanup() {
+ if (state != State.CLEANED) {
+ try {
+ // Close the stream quietly.
+ CommonUtils.runQuietly(out::close, false);
+
+ // Delete the staging file if exists.
+ Files.deleteIfExists(path);
+ } catch (Exception e) {
+ LOG.error("Failed to delete staging file, stagingFile: {}", path, e);
+ } finally {
+ state = State.CLEANED;
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("path", path)
+ .add("stagingBufferSize", stagingBufferSize)
+ .add("wroteByteSize", size())
+ .toString();
+ }
+
+ private static class StagingFileOutputStream extends OutputStream {
+ private final Path path;
+ private byte[] buffer;
+ private boolean memBuffered;
+ private int writePos;
+ private OutputStream out;
+
+ private StagingFileOutputStream(Path path, int stagingBufferSize) {
+ this.path = path;
+ this.buffer = new byte[stagingBufferSize];
+ this.memBuffered = true;
+ this.writePos = 0;
+ }
+
+ private int size() {
+ return writePos;
+ }
+
+ public InputStream newIn() {
+ // Just wrap it as a byte array input stream if the staging bytes are still in the in-memory buffer.
+ if (memBuffered) {
+ return new ByteArrayInputStream(buffer, 0, writePos);
+ }
+
+ // Create a buffered file input stream.
+ try {
+ return new BufferedInputStream(Files.newInputStream(path));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ write(new byte[]{(byte) b}, 0, 1);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ if (memBuffered && writePos + len > buffer.length) {
+ flushMemToFile();
+ }
+
+ if (memBuffered) {
+ System.arraycopy(b, off, buffer, writePos, len);
+ } else {
+ out.write(b, off, len);
+ }
+
+ writePos += len;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (out != null) {
+ out.close();
+ out = null;
+ }
+ }
+
+ private void flushMemToFile() throws IOException {
+ // Flush the buffered data to the new file OutputStream.
+ out = new BufferedOutputStream(Files.newOutputStream(path));
+ out.write(buffer, 0, writePos);
+ memBuffered = false;
+ buffer = null;
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/StagingPart.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/StagingPart.java
new file mode 100644
index 00000000000..1f8ef4bc092
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/StagingPart.java
@@ -0,0 +1,79 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.staging;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public interface StagingPart {
+
+ /**
+ * Write bytes into the staging part.
+ *
+ * @param b the buffer to write.
+ * @throws IOException if any IO error.
+ */
+ default void write(byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ /**
+ * Write the bytes into the staging part.
+ *
+ * @param b the buffer to write.
+ * @param off the start offset in buffer.
+ * @param len the length.
+ * @throws IOException if any IO error.
+ */
+ void write(byte[] b, int off, int len) throws IOException;
+
+ /**
+ * Complete the writing process and cannot write more bytes once we've completed this part.
+ *
+ * @throws IOException if any IO error.
+ */
+ void complete() throws IOException;
+
+ /**
+ * The wrote size of staging part.
+ *
+ * @return the staging part size.
+ */
+ long size();
+
+ /**
+ * Access the {@link State} of this part.
+ *
+ * @return the {@link State}.
+ */
+ State state();
+
+ /**
+ * Create a separate new {@link InputStream} to read the staging part data once we've completed the
+ * writing by calling {@link StagingPart#complete()} . Call this method several times will return
+ * many {@link InputStream}s, and remember to close the newly created stream.
+ *
+ * @return a totally new {@link InputStream}.
+ */
+ InputStream newIn();
+
+ /**
+ * Clean all the {@link StagingPart}'s resources, such as removing temporary file, free the buffered data etc. it
+ * should be idempotent and quiet (without throwing IO error).
+ */
+ void cleanup();
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/State.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/State.java
new file mode 100644
index 00000000000..418baa6d9b1
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/State.java
@@ -0,0 +1,23 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.staging;
+
+public enum State {
+ WRITABLE,
+ READABLE,
+ CLEANED
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/tos/ChainTOSInputStream.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/tos/ChainTOSInputStream.java
new file mode 100644
index 00000000000..3a54410080c
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/tos/ChainTOSInputStream.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.tos;
+
+import org.apache.hadoop.fs.tosfs.common.Chain;
+import org.apache.hadoop.util.Preconditions;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class ChainTOSInputStream extends InputStream {
+ private final Chain<TOSInputStream> chain;
+ private final TOS.GetObjectFactory factory;
+ private final String key;
+ private long curOff;
+ private final long endOff; // range end offset (inclusive)
+ private final long maxDrainByteSize;
+ private final int maxInputStreamRetries;
+
+ private int readBytes;
+ private long skipped;
+ private byte[] objChecksum = null;
+ private final AtomicBoolean closed = new AtomicBoolean(false);
+
+ public ChainTOSInputStream(
+ TOS.GetObjectFactory factory,
+ String key,
+ long startOff,
+ long endOff,
+ long maxDrainByteSize,
+ int maxInputStreamRetries) {
+ this.factory = factory;
+ this.key = key;
+ this.curOff = startOff;
+ this.endOff = endOff;
+ this.maxDrainByteSize = maxDrainByteSize;
+ this.maxInputStreamRetries = maxInputStreamRetries;
+ this.chain = createChain();
+ Preconditions.checkNotNull(objChecksum, "Checksum should not be null.");
+ }
+
+ private Chain<TOSInputStream> createChain() {
+ Chain.Builder<TOSInputStream> builder = Chain.<TOSInputStream>builder()
+ .shouldContinue(e -> !(e instanceof EOFException));
+
+ for (int i = 0; i <= maxInputStreamRetries; i++) {
+ builder.addLast(() -> {
+ GetObjectOutput output = factory.create(key, curOff, endOff);
+
+ // Note: If there are some IO errors occur, the ChainTOSInputStream will create a new stream in the chain to
+ // continue reading object data, we need to record the checksum during first open object stream, and ensure the
+ // checksum of object stream won't be changed if opening object many times within the lifecycle of the chained
+ // stream in case the underlying object is changed.
+ if (objChecksum == null) {
+ // Init the stream checksum.
+ objChecksum = output.checksum();
+ }
+ return new TOSInputStream(output, curOff, endOff, maxDrainByteSize, objChecksum);
+ });
+ }
+
+ try {
+ return builder.build();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ skipped = 0;
+ return chain.run(stream -> {
+ long skip = stream.skip(n - skipped);
+
+ curOff += skip;
+ skipped += skip;
+ return skipped;
+ });
+ }
+
+ @Override
+ public int read() throws IOException {
+ return chain.run(stream -> {
+ int ret = stream.read();
+ curOff++;
+ return ret;
+ });
+ }
+
+ @Override
+ public int available() throws IOException {
+ return chain.run(InputStream::available);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ readBytes = 0;
+ return chain.run(in -> {
+ int read = in.read(b, off + readBytes, len - readBytes);
+
+ readBytes += read;
+ curOff += read;
+ return readBytes;
+ });
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed.compareAndSet(false, true)) {
+ chain.close();
+ }
+ }
+
+ public byte[] checksum() {
+ return objChecksum;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/tos/DelegationClient.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/tos/DelegationClient.java
new file mode 100644
index 00000000000..066473f3f2f
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/tos/DelegationClient.java
@@ -0,0 +1,1230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.tos;
+
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.TOSClientConfiguration;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.TOSV2ClientBuilder;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.TosClientException;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.TosException;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.TosServerException;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.auth.Credential;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.auth.Credentials;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.TOSV2;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.comm.HttpStatus;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.comm.common.ACLType;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.internal.RequestOptionsBuilder;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.acl.GetObjectAclOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.acl.PutObjectAclInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.acl.PutObjectAclOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.CreateBucketInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.CreateBucketOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.CreateBucketV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.CreateBucketV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketCORSInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketCORSOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketCustomDomainInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketCustomDomainOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketEncryptionInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketEncryptionOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketInventoryInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketInventoryOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketLifecycleInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketLifecycleOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketMirrorBackInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketMirrorBackOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketPolicyInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketPolicyOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketRealTimeLogInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketRealTimeLogOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketRenameInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketRenameOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketReplicationInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketReplicationOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketTaggingInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketTaggingOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketWebsiteInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.DeleteBucketWebsiteOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketACLInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketACLOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketCORSInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketCORSOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketEncryptionInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketEncryptionOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketInventoryInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketInventoryOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketLifecycleInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketLifecycleOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketLocationInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketLocationOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketMirrorBackInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketMirrorBackOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketNotificationInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketNotificationOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketNotificationType2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketNotificationType2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketPolicyInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketPolicyOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketRealTimeLogInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketRealTimeLogOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketRenameInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketRenameOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketReplicationInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketReplicationOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketTaggingInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketTaggingOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketVersioningInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketVersioningOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketWebsiteInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.GetBucketWebsiteOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.HeadBucketOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.HeadBucketV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.HeadBucketV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.ListBucketCustomDomainInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.ListBucketCustomDomainOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.ListBucketInventoryInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.ListBucketInventoryOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.ListBucketsInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.ListBucketsOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.ListBucketsV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.ListBucketsV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketACLInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketACLOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketCORSInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketCORSOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketCustomDomainInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketCustomDomainOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketEncryptionInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketEncryptionOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketInventoryInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketInventoryOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketLifecycleInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketLifecycleOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketMirrorBackInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketMirrorBackOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketNotificationInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketNotificationOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketNotificationType2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketNotificationType2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketPolicyInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketPolicyOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketRealTimeLogInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketRealTimeLogOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketRenameInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketRenameOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketReplicationInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketReplicationOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketStorageClassInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketStorageClassOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketTaggingInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketTaggingOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketVersioningInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketVersioningOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketWebsiteInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.bucket.PutBucketWebsiteOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.AbortMultipartUploadInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.AbortMultipartUploadOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.AppendObjectInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.AppendObjectOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.CompleteMultipartUploadInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.CompleteMultipartUploadOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.CompleteMultipartUploadV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.CompleteMultipartUploadV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.CopyObjectOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.CopyObjectV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.CopyObjectV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.CreateMultipartUploadInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.CreateMultipartUploadOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.DeleteMultiObjectsInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.DeleteMultiObjectsOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.DeleteMultiObjectsV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.DeleteMultiObjectsV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.DeleteObjectInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.DeleteObjectOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.DeleteObjectTaggingInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.DeleteObjectTaggingOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.DownloadFileInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.DownloadFileOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.FetchObjectInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.FetchObjectOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetFetchTaskInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetFetchTaskOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetFileStatusInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetFileStatusOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetObjectACLV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetObjectACLV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetObjectOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetObjectTaggingInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetObjectTaggingOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetObjectToFileInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetObjectToFileOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetObjectV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetObjectV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetSymlinkInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.GetSymlinkOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.HeadObjectOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.HeadObjectV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.HeadObjectV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListMultipartUploadsInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListMultipartUploadsOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListMultipartUploadsV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListMultipartUploadsV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListObjectVersionsInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListObjectVersionsOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListObjectVersionsV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListObjectVersionsV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListObjectsInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListObjectsOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListObjectsType2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListObjectsType2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListObjectsV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListObjectsV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListPartsInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListPartsOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListUploadedPartsInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ListUploadedPartsOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ObjectMetaRequestOptions;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PreSignedPolicyURLInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PreSignedPolicyURLOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PreSignedPostSignatureInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PreSignedPostSignatureOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PreSignedURLInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PreSignedURLOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PreSingedPolicyURLInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PreSingedPolicyURLOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PutFetchTaskInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PutFetchTaskOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PutObjectACLInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PutObjectACLOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PutObjectFromFileInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PutObjectFromFileOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PutObjectInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PutObjectOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PutObjectTaggingInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PutObjectTaggingOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PutSymlinkInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.PutSymlinkOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.RenameObjectInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.RenameObjectOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.RestoreObjectInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.RestoreObjectOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ResumableCopyObjectInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.ResumableCopyObjectOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.SetObjectMetaInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.SetObjectMetaOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadFileInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadFileOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadFileV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadFileV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadPartCopyInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadPartCopyOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadPartCopyV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadPartCopyV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadPartFromFileInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadPartFromFileOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadPartInput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadPartOutput;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadPartV2Input;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.model.object.UploadPartV2Output;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.transport.TransportConfig;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.fs.tosfs.object.InputStreamProvider;
+import org.apache.hadoop.fs.tosfs.object.Part;
+import org.apache.hadoop.fs.tosfs.util.RetryableUtils;
+import org.apache.hadoop.thirdparty.com.google.common.base.Throwables;
+import org.apache.hadoop.thirdparty.com.google.common.io.CountingInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
+import java.time.Duration;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.Callable;
+import javax.net.ssl.SSLException;
+
+public class DelegationClient implements TOSV2 {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DelegationClient.class);
+
+ private final Credentials provider;
+ private final TOSClientConfiguration config;
+ private int maxRetryTimes;
+ private TOSV2 client;
+ private volatile Credential usedCredential;
+ private final List<String> nonRetryable409ErrorCodes;
+
+ protected DelegationClient(
+ TOSClientConfiguration configuration, int maxRetryTimes, List<String> nonRetryable409ErrorCodes) {
+ this.config = configuration;
+ this.maxRetryTimes = maxRetryTimes;
+ this.provider = configuration.getCredentials();
+ this.usedCredential = provider.credential();
+ this.client = new TOSV2ClientBuilder().build(configuration);
+ this.nonRetryable409ErrorCodes = nonRetryable409ErrorCodes;
+ }
+
+ @VisibleForTesting
+ void setClient(TOSV2 client) {
+ this.client = client;
+ }
+
+ public TOSV2 client() {
+ return client;
+ }
+
+ @VisibleForTesting
+ void setMaxRetryTimes(int maxRetryTimes) {
+ this.maxRetryTimes = maxRetryTimes;
+ }
+
+ public int maxRetryTimes() {
+ return maxRetryTimes;
+ }
+
+ public TOSClientConfiguration config() {
+ return config;
+ }
+
+ public Credential usedCredential() {
+ return usedCredential;
+ }
+
+ @Override
+ public CreateBucketV2Output createBucket(String bucket) throws TosException {
+ return retry(() -> client.createBucket(bucket));
+ }
+
+ @Override
+ public CreateBucketV2Output createBucket(CreateBucketV2Input input) throws TosException {
+ return retry(() -> client.createBucket(input));
+ }
+
+ @Override
+ public HeadBucketV2Output headBucket(HeadBucketV2Input input) throws TosException {
+ return retry(() -> client.headBucket(input));
+ }
+
+ @Override
+ public DeleteBucketOutput deleteBucket(DeleteBucketInput input) throws TosException {
+ return retry(() -> client.deleteBucket(input));
+ }
+
+ @Override
+ public ListBucketsV2Output listBuckets(ListBucketsV2Input input) throws TosException {
+ return retry(() -> client.listBuckets(input));
+ }
+
+ @Override
+ public CreateBucketOutput createBucket(CreateBucketInput input) throws TosException {
+ return retry(() -> client.createBucket(input));
+ }
+
+ @Override
+ public HeadBucketOutput headBucket(String bucket) throws TosException {
+ return retry(() -> client.headBucket(bucket));
+ }
+
+ @Override
+ public DeleteBucketOutput deleteBucket(String bucket) throws TosException {
+ return retry(() -> client.deleteBucket(bucket));
+ }
+
+ @Override
+ public ListBucketsOutput listBuckets(ListBucketsInput input) throws TosException {
+ return retry(() -> client.listBuckets(input));
+ }
+
+ @Override
+ public PutBucketPolicyOutput putBucketPolicy(String bucket, String policy) throws TosException {
+ return retry(() -> client.putBucketPolicy(bucket, policy));
+ }
+
+ @Override
+ public PutBucketPolicyOutput putBucketPolicy(PutBucketPolicyInput input) throws TosException {
+ return retry(() -> client.putBucketPolicy(input));
+ }
+
+ @Override
+ public GetBucketPolicyOutput getBucketPolicy(String bucket) throws TosException {
+ return retry(() -> client.getBucketPolicy(bucket));
+ }
+
+ @Override
+ public GetBucketPolicyOutput getBucketPolicy(GetBucketPolicyInput input) throws TosException {
+ return retry(() -> client.getBucketPolicy(input));
+ }
+
+ @Override
+ public DeleteBucketPolicyOutput deleteBucketPolicy(String bucket) throws TosException {
+ return retry(() -> client.deleteBucketPolicy(bucket));
+ }
+
+ @Override
+ public GetObjectOutput getObject(String bucket, String objectKey,
+ RequestOptionsBuilder... builders) throws TosException {
+ return retry(() -> client.getObject(bucket, objectKey, builders));
+ }
+
+ @Override
+ public HeadObjectOutput headObject(String bucket, String objectKey,
+ RequestOptionsBuilder... builders) throws TosException {
+ return retry(() -> client.headObject(bucket, objectKey, builders));
+ }
+
+ @Override
+ public DeleteObjectOutput deleteObject(String bucket, String objectKey,
+ RequestOptionsBuilder... builders) throws TosException {
+ return retry(() -> client.deleteObject(bucket, objectKey, builders));
+ }
+
+ @Override
+ public DeleteMultiObjectsOutput deleteMultiObjects(
+ String bucket,
+ DeleteMultiObjectsInput input,
+ RequestOptionsBuilder... builders)
+ throws TosException {
+ return retry(() -> client.deleteMultiObjects(bucket, input, builders));
+ }
+
+ @Override
+ public PutObjectOutput putObject(
+ String bucket, String objectKey, InputStream inputStream,
+ RequestOptionsBuilder... builders)
+ throws TosException {
+ throw new UnsupportedOperationException("Not supported");
+ }
+
+ @Override
+ public UploadFileOutput uploadFile(
+ String bucket, UploadFileInput input,
+ RequestOptionsBuilder... builders) throws TosException {
+ return retry(() -> client.uploadFile(bucket, input, builders));
+ }
+
+ @Override
+ public AppendObjectOutput appendObject(
+ String bucket, String objectKey, InputStream content, long offset,
+ RequestOptionsBuilder... builders)
+ throws TosException {
+ throw new UnsupportedOperationException("Not supported");
+ }
+
+ @Override
+ public SetObjectMetaOutput setObjectMeta(String bucket, String objectKey, RequestOptionsBuilder... builders)
+ throws TosException {
+ return retry(() -> client.setObjectMeta(bucket, objectKey, builders));
+ }
+
+ @Override
+ public ListObjectsOutput listObjects(String bucket, ListObjectsInput input) throws TosException {
+ return retry(() -> client.listObjects(bucket, input));
+ }
+
+ @Override
+ public ListObjectVersionsOutput listObjectVersions(String bucket, ListObjectVersionsInput input) throws TosException {
+ return retry(() -> client.listObjectVersions(bucket, input));
+ }
+
+ @Override
+ public CopyObjectOutput copyObject(
+ String bucket, String srcObjectKey, String dstObjectKey,
+ RequestOptionsBuilder... builders)
+ throws TosException {
+ return retry(() -> client.copyObject(bucket, srcObjectKey, dstObjectKey, builders));
+ }
+
+ @Override
+ public CopyObjectOutput copyObjectTo(
+ String bucket, String dstBucket, String dstObjectKey,
+ String srcObjectKey,
+ RequestOptionsBuilder... builders)
+ throws TosException {
+ return retry(() -> client.copyObjectTo(bucket, dstBucket, dstObjectKey, srcObjectKey, builders));
+ }
+
+ @Override
+ public CopyObjectOutput copyObjectFrom(
+ String bucket, String srcBucket, String srcObjectKey, String dstObjectKey,
+ RequestOptionsBuilder... builders)
+ throws TosException {
+ return retry(() -> client.copyObjectFrom(bucket, srcBucket, srcObjectKey, dstObjectKey, builders));
+ }
+
+ @Override
+ public UploadPartCopyOutput uploadPartCopy(
+ String bucket, UploadPartCopyInput input,
+ RequestOptionsBuilder... builders) throws TosException {
+ return retry(() -> client.uploadPartCopy(bucket, input, builders));
+ }
+
+ @Override
+ public PutObjectAclOutput putObjectAcl(String bucket, PutObjectAclInput input) throws TosException {
+ return retry(() -> client.putObjectAcl(bucket, input));
+ }
+
+ @Override
+ public GetObjectAclOutput getObjectAcl(
+ String bucket, String objectKey,
+ RequestOptionsBuilder... builders)
+ throws TosException {
+ return retry(() -> client.getObjectAcl(bucket, objectKey, builders));
+ }
+
+ @Override
+ public CreateMultipartUploadOutput createMultipartUpload(
+ String bucket, String objectKey,
+ RequestOptionsBuilder... builders)
+ throws TosException {
+ return retry(() -> client.createMultipartUpload(bucket, objectKey, builders));
+ }
+
+ @Override
+ public UploadPartOutput uploadPart(
+ String bucket, UploadPartInput input,
+ RequestOptionsBuilder... builders)
+ throws TosException {
+ throw new UnsupportedOperationException("Not supported");
+ }
+
+ @Override
+ public CompleteMultipartUploadOutput completeMultipartUpload(
+ String bucket,
+ CompleteMultipartUploadInput input)
+ throws TosException {
+ return retry(() -> client.completeMultipartUpload(bucket, input));
+ }
+
+ @Override
+ public AbortMultipartUploadOutput abortMultipartUpload(
+ String bucket,
+ AbortMultipartUploadInput input)
+ throws TosException {
+ return retry(() -> client.abortMultipartUpload(bucket, input));
+ }
+
+ @Override
+ public ListUploadedPartsOutput listUploadedParts(
+ String bucket,
+ ListUploadedPartsInput input,
+ RequestOptionsBuilder... builders)
+ throws TosException {
+ return retry(() -> client.listUploadedParts(bucket, input, builders));
+ }
+
+ @Override
+ public ListMultipartUploadsOutput listMultipartUploads(
+ String bucket,
+ ListMultipartUploadsInput input)
+ throws TosException {
+ return retry(() -> client.listMultipartUploads(bucket, input));
+ }
+
+ @Override
+ public String preSignedURL(
+ String httpMethod, String bucket, String objectKey, Duration ttl,
+ RequestOptionsBuilder... builders)
+ throws TosException {
+ return retry(() -> client.preSignedURL(httpMethod, bucket, objectKey, ttl, builders));
+ }
+
+ @Override
+ public DeleteBucketPolicyOutput deleteBucketPolicy(DeleteBucketPolicyInput input)
+ throws TosException {
+ return retry(() -> client.deleteBucketPolicy(input));
+ }
+
+ @Override
+ public PutBucketCORSOutput putBucketCORS(PutBucketCORSInput input)
+ throws TosException {
+ return retry(() -> client.putBucketCORS(input));
+ }
+
+ @Override
+ public GetBucketCORSOutput getBucketCORS(GetBucketCORSInput input)
+ throws TosException {
+ return retry(() -> client.getBucketCORS(input));
+ }
+
+ @Override
+ public DeleteBucketCORSOutput deleteBucketCORS(DeleteBucketCORSInput input)
+ throws TosException {
+ return retry(() -> client.deleteBucketCORS(input));
+ }
+
+ @Override
+ public PutBucketStorageClassOutput putBucketStorageClass(PutBucketStorageClassInput input)
+ throws TosException {
+ return retry(() -> client.putBucketStorageClass(input));
+ }
+
+ @Override
+ public GetBucketLocationOutput getBucketLocation(GetBucketLocationInput input)
+ throws TosException {
+ return retry(() -> client.getBucketLocation(input));
+ }
+
+ @Override
+ public PutBucketLifecycleOutput putBucketLifecycle(PutBucketLifecycleInput input)
+ throws TosException {
+ return retry(() -> client.putBucketLifecycle(input));
+ }
+
+ @Override
+ public GetBucketLifecycleOutput getBucketLifecycle(GetBucketLifecycleInput input)
+ throws TosException {
+ return retry(() -> client.getBucketLifecycle(input));
+ }
+
+ @Override
+ public DeleteBucketLifecycleOutput deleteBucketLifecycle(DeleteBucketLifecycleInput input)
+ throws TosException {
+ return retry(() -> client.deleteBucketLifecycle(input));
+ }
+
+ @Override
+ public PutBucketMirrorBackOutput putBucketMirrorBack(PutBucketMirrorBackInput input)
+ throws TosException {
+ return retry(() -> client.putBucketMirrorBack(input));
+ }
+
+ @Override
+ public GetBucketMirrorBackOutput getBucketMirrorBack(GetBucketMirrorBackInput input)
+ throws TosException {
+ return retry(() -> client.getBucketMirrorBack(input));
+ }
+
+ @Override
+ public DeleteBucketMirrorBackOutput deleteBucketMirrorBack(DeleteBucketMirrorBackInput input)
+ throws TosException {
+ return retry(() -> client.deleteBucketMirrorBack(input));
+ }
+
+ @Override
+ public PutBucketReplicationOutput putBucketReplication(PutBucketReplicationInput input)
+ throws TosException {
+ return retry(() -> client.putBucketReplication(input));
+ }
+
+ @Override
+ public GetBucketReplicationOutput getBucketReplication(GetBucketReplicationInput input)
+ throws TosException {
+ return retry(() -> client.getBucketReplication(input));
+ }
+
+ @Override
+ public DeleteBucketReplicationOutput deleteBucketReplication(DeleteBucketReplicationInput input)
+ throws TosException {
+ return retry(() -> client.deleteBucketReplication(input));
+ }
+
+ @Override
+ public PutBucketVersioningOutput putBucketVersioning(PutBucketVersioningInput input)
+ throws TosException {
+ return retry(() -> client.putBucketVersioning(input));
+ }
+
+ @Override
+ public GetBucketVersioningOutput getBucketVersioning(GetBucketVersioningInput input)
+ throws TosException {
+ return retry(() -> client.getBucketVersioning(input));
+ }
+
+ @Override
+ public PutBucketWebsiteOutput putBucketWebsite(PutBucketWebsiteInput input)
+ throws TosException {
+ return retry(() -> client.putBucketWebsite(input));
+ }
+
+ @Override
+ public GetBucketWebsiteOutput getBucketWebsite(GetBucketWebsiteInput input)
+ throws TosException {
+ return retry(() -> client.getBucketWebsite(input));
+ }
+
+ @Override
+ public DeleteBucketWebsiteOutput deleteBucketWebsite(DeleteBucketWebsiteInput input)
+ throws TosException {
+ return retry(() -> client.deleteBucketWebsite(input));
+ }
+
+ @Override
+ public PutBucketNotificationOutput putBucketNotification(PutBucketNotificationInput input)
+ throws TosException {
+ return retry(() -> client.putBucketNotification(input));
+ }
+
+ @Override
+ public GetBucketNotificationOutput getBucketNotification(GetBucketNotificationInput input)
+ throws TosException {
+ return retry(() -> client.getBucketNotification(input));
+ }
+
+ @Override
+ public PutBucketNotificationType2Output putBucketNotificationType2(PutBucketNotificationType2Input input)
+ throws TosException {
+ return retry(() -> client.putBucketNotificationType2(input));
+ }
+
+ @Override
+ public GetBucketNotificationType2Output getBucketNotificationType2(GetBucketNotificationType2Input input)
+ throws TosException {
+ return retry(() -> client.getBucketNotificationType2(input));
+ }
+
+ @Override
+ public PutBucketCustomDomainOutput putBucketCustomDomain(PutBucketCustomDomainInput input)
+ throws TosException {
+ return retry(() -> client.putBucketCustomDomain(input));
+ }
+
+ @Override
+ public ListBucketCustomDomainOutput listBucketCustomDomain(ListBucketCustomDomainInput input)
+ throws TosException {
+ return retry(() -> client.listBucketCustomDomain(input));
+ }
+
+ @Override
+ public DeleteBucketCustomDomainOutput deleteBucketCustomDomain(DeleteBucketCustomDomainInput input)
+ throws TosException {
+ return retry(() -> client.deleteBucketCustomDomain(input));
+ }
+
+ @Override
+ public PutBucketRealTimeLogOutput putBucketRealTimeLog(PutBucketRealTimeLogInput input)
+ throws TosException {
+ return retry(() -> client.putBucketRealTimeLog(input));
+ }
+
+ @Override
+ public GetBucketRealTimeLogOutput getBucketRealTimeLog(GetBucketRealTimeLogInput input)
+ throws TosException {
+ return retry(() -> client.getBucketRealTimeLog(input));
+ }
+
+ @Override
+ public DeleteBucketRealTimeLogOutput deleteBucketRealTimeLog(DeleteBucketRealTimeLogInput input)
+ throws TosException {
+ return retry(() -> deleteBucketRealTimeLog(input));
+ }
+
+ @Override
+ public PutBucketACLOutput putBucketACL(PutBucketACLInput input) throws TosException {
+ return retry(() -> client.putBucketACL(input));
+ }
+
+ @Override
+ public GetBucketACLOutput getBucketACL(GetBucketACLInput input) throws TosException {
+ return retry(() -> client.getBucketACL(input));
+ }
+
+ @Override
+ public PutBucketRenameOutput putBucketRename(PutBucketRenameInput input) throws TosException {
+ return retry(() -> client.putBucketRename(input));
+ }
+
+ @Override
+ public GetBucketRenameOutput getBucketRename(GetBucketRenameInput input) throws TosException {
+ return retry(() -> client.getBucketRename(input));
+ }
+
+ @Override
+ public DeleteBucketRenameOutput deleteBucketRename(DeleteBucketRenameInput input) throws TosException {
+ return retry(() -> client.deleteBucketRename(input));
+ }
+
+ @Override
+ public PutBucketEncryptionOutput putBucketEncryption(PutBucketEncryptionInput input) throws TosException {
+ return retry(() -> client.putBucketEncryption(input));
+ }
+
+ @Override
+ public GetBucketEncryptionOutput getBucketEncryption(GetBucketEncryptionInput input) throws TosException {
+ return retry(() -> client.getBucketEncryption(input));
+ }
+
+ @Override
+ public DeleteBucketEncryptionOutput deleteBucketEncryption(DeleteBucketEncryptionInput input) throws TosException {
+ return retry(() -> client.deleteBucketEncryption(input));
+ }
+
+ @Override
+ public PutBucketTaggingOutput putBucketTagging(PutBucketTaggingInput input) throws TosException {
+ return retry(() -> client.putBucketTagging(input));
+ }
+
+ @Override
+ public GetBucketTaggingOutput getBucketTagging(GetBucketTaggingInput input) throws TosException {
+ return retry(() -> client.getBucketTagging(input));
+ }
+
+ @Override
+ public DeleteBucketTaggingOutput deleteBucketTagging(DeleteBucketTaggingInput input) throws TosException {
+ return retry(() -> client.deleteBucketTagging(input));
+ }
+
+ @Override
+ public PutBucketInventoryOutput putBucketInventory(PutBucketInventoryInput input)
+ throws TosException {
+ return retry(() -> client.putBucketInventory(input));
+ }
+
+ @Override
+ public GetBucketInventoryOutput getBucketInventory(GetBucketInventoryInput input) throws TosException {
+ return retry(() -> client.getBucketInventory(input));
+ }
+
+ @Override
+ public ListBucketInventoryOutput listBucketInventory(ListBucketInventoryInput input) throws TosException {
+ return retry(() -> client.listBucketInventory(input));
+ }
+
+ @Override
+ public DeleteBucketInventoryOutput deleteBucketInventory(DeleteBucketInventoryInput input) throws TosException {
+ return retry(() -> client.deleteBucketInventory(input));
+ }
+
+ @Override
+ public GetObjectV2Output getObject(GetObjectV2Input input) throws TosException {
+ return retry(() -> client.getObject(input));
+ }
+
+ @Override
+ public GetObjectToFileOutput getObjectToFile(GetObjectToFileInput input) throws TosException {
+ return retry(() -> client.getObjectToFile(input));
+ }
+
+ @Override
+ public GetFileStatusOutput getFileStatus(GetFileStatusInput input) throws TosException {
+ return retry(() -> client.getFileStatus(input));
+ }
+
+ @Override
+ public UploadFileV2Output uploadFile(UploadFileV2Input input) throws TosException {
+ return retry(() -> client.uploadFile(input));
+ }
+
+ @Override
+ public DownloadFileOutput downloadFile(DownloadFileInput input) throws TosException {
+ return retry(() -> client.downloadFile(input));
+ }
+
+ @Override
+ public ResumableCopyObjectOutput resumableCopyObject(ResumableCopyObjectInput input)
+ throws TosException {
+ return retry(() -> client.resumableCopyObject(input));
+ }
+
+ @Override
+ public HeadObjectV2Output headObject(HeadObjectV2Input input) throws TosException {
+ return retry(() -> client.headObject(input));
+ }
+
+ @Override
+ public DeleteObjectOutput deleteObject(DeleteObjectInput input) throws TosException {
+ return retry(() -> client.deleteObject(input));
+ }
+
+ @Override
+ public DeleteMultiObjectsV2Output deleteMultiObjects(DeleteMultiObjectsV2Input input)
+ throws TosException {
+ return retry(() -> client.deleteMultiObjects(input));
+ }
+
+ public PutObjectOutput put(
+ String bucket, String key, InputStreamProvider streamProvider,
+ long contentLength, ACLType aclType) {
+ return retry(() -> client.putObject(newPutObjectRequest(bucket, key, streamProvider, contentLength, aclType)));
+ }
+
+ private PutObjectInput newPutObjectRequest(
+ String bucket,
+ String key,
+ InputStreamProvider streamProvider,
+ long contentLength,
+ ACLType aclType) {
+
+ return PutObjectInput.builder()
+ .bucket(bucket)
+ .key(key)
+ .content(streamProvider.newStream())
+ .contentLength(contentLength)
+ .options(new ObjectMetaRequestOptions()
+ .setAclType(aclType))
+ .build();
+ }
+
+ public AppendObjectOutput appendObject(String bucket, String key, InputStreamProvider streamProvider,
+ long offset, long contentLength, String originalCrc64, ACLType aclType) {
+ // originalCrc64 is needed when appending data to object. It should be the object's crc64 checksum if the object
+ // exists, and null if the object doesn't exist.
+ return retry(() -> client.appendObject(
+ newAppendObjectRequest(bucket, key, streamProvider, offset, contentLength, originalCrc64, aclType)));
+ }
+
+ private AppendObjectInput newAppendObjectRequest(
+ String bucket,
+ String key,
+ InputStreamProvider streamProvider,
+ long offset,
+ long contentLength,
+ String preCrc64ecma,
+ ACLType aclType) {
+ return AppendObjectInput.builder()
+ .bucket(bucket)
+ .key(key)
+ .content(streamProvider.newStream())
+ .offset(offset)
+ .contentLength(contentLength)
+ .preHashCrc64ecma(preCrc64ecma)
+ .options(new ObjectMetaRequestOptions()
+ .setAclType(aclType))
+ .build();
+ }
+
+ @Override
+ public PutObjectOutput putObject(PutObjectInput input) throws TosException {
+ throw new UnsupportedOperationException("Not supported");
+ }
+
+ @Override
+ public PutObjectFromFileOutput putObjectFromFile(PutObjectFromFileInput input)
+ throws TosException {
+ return retry(() -> client.putObjectFromFile(input));
+ }
+
+ @Override
+ public AppendObjectOutput appendObject(AppendObjectInput input)
+ throws TosException {
+ throw new UnsupportedOperationException("Not supported");
+ }
+
+ @Override
+ public SetObjectMetaOutput setObjectMeta(SetObjectMetaInput input)
+ throws TosException {
+ return retry(() -> client.setObjectMeta(input));
+ }
+
+ @Override
+ public ListObjectsV2Output listObjects(ListObjectsV2Input input)
+ throws TosException {
+ return retry(() -> client.listObjects(input));
+ }
+
+ @Override
+ public ListObjectsType2Output listObjectsType2(ListObjectsType2Input input)
+ throws TosException {
+ return retry(() -> client.listObjectsType2(input));
+ }
+
+ @Override
+ public ListObjectVersionsV2Output listObjectVersions(ListObjectVersionsV2Input input)
+ throws TosException {
+ return retry(() -> client.listObjectVersions(input));
+ }
+
+ @Override
+ public CopyObjectV2Output copyObject(CopyObjectV2Input input)
+ throws TosException {
+ return retry(() -> client.copyObject(input));
+ }
+
+ @Override
+ public UploadPartCopyV2Output uploadPartCopy(UploadPartCopyV2Input input)
+ throws TosException {
+ return retry(() -> client.uploadPartCopy(input));
+ }
+
+ @Override
+ public PutObjectACLOutput putObjectAcl(PutObjectACLInput input)
+ throws TosException {
+ return retry(() -> client.putObjectAcl(input));
+ }
+
+ @Override
+ public GetObjectACLV2Output getObjectAcl(GetObjectACLV2Input input)
+ throws TosException {
+ return retry(() -> client.getObjectAcl(input));
+ }
+
+ @Override
+ public PutObjectTaggingOutput putObjectTagging(PutObjectTaggingInput input)
+ throws TosException {
+ return retry(() -> client.putObjectTagging(input));
+ }
+
+ @Override
+ public GetObjectTaggingOutput getObjectTagging(GetObjectTaggingInput input)
+ throws TosException {
+ return retry(() -> client.getObjectTagging(input));
+ }
+
+ @Override
+ public DeleteObjectTaggingOutput deleteObjectTagging(DeleteObjectTaggingInput input)
+ throws TosException {
+ return retry(() -> client.deleteObjectTagging(input));
+ }
+
+ @Override
+ public FetchObjectOutput fetchObject(FetchObjectInput input) throws TosException {
+ return retry(() -> client.fetchObject(input));
+ }
+
+ @Override
+ public PutFetchTaskOutput putFetchTask(PutFetchTaskInput input) throws TosException {
+ return retry(() -> client.putFetchTask(input));
+ }
+
+ @Override
+ public GetFetchTaskOutput getFetchTask(GetFetchTaskInput input) throws TosException {
+ return retry(() -> client.getFetchTask(input));
+ }
+
+ @Override
+ public CreateMultipartUploadOutput createMultipartUpload(CreateMultipartUploadInput input)
+ throws TosException {
+ return retry(() -> client.createMultipartUpload(input));
+ }
+
+ public Part uploadPart(
+ String bucket,
+ String key,
+ String uploadId,
+ int partNum,
+ InputStreamProvider streamProvider,
+ long contentLength,
+ ACLType aclType) {
+ return retry(() -> {
+ InputStream in = streamProvider.newStream();
+ CountingInputStream countedIn = new CountingInputStream(in);
+ UploadPartV2Input request = UploadPartV2Input.builder()
+ .bucket(bucket)
+ .key(key)
+ .partNumber(partNum)
+ .uploadID(uploadId)
+ .content(countedIn)
+ .contentLength(contentLength)
+ .options(new ObjectMetaRequestOptions()
+ .setAclType(aclType))
+ .build();
+ UploadPartV2Output output = client.uploadPart(request);
+ return new Part(output.getPartNumber(), countedIn.getCount(), output.getEtag());
+ });
+ }
+
+ @Override
+ public UploadPartV2Output uploadPart(UploadPartV2Input input) throws TosException {
+ throw new UnsupportedOperationException("Not supported");
+ }
+
+ @Override
+ public UploadPartFromFileOutput uploadPartFromFile(UploadPartFromFileInput input)
+ throws TosException {
+ return retry(() -> client.uploadPartFromFile(input));
+ }
+
+ @Override
+ public CompleteMultipartUploadV2Output completeMultipartUpload(CompleteMultipartUploadV2Input input)
+ throws TosException {
+ return retry(() -> client.completeMultipartUpload(input));
+ }
+
+ @Override
+ public AbortMultipartUploadOutput abortMultipartUpload(AbortMultipartUploadInput input)
+ throws TosException {
+ return retry(() -> client.abortMultipartUpload(input));
+ }
+
+ @Override
+ public ListPartsOutput listParts(ListPartsInput input) throws TosException {
+ return retry(() -> client.listParts(input));
+ }
+
+ @Override
+ public ListMultipartUploadsV2Output listMultipartUploads(ListMultipartUploadsV2Input input)
+ throws TosException {
+ return retry(() -> client.listMultipartUploads(input));
+ }
+
+ @Override
+ public RenameObjectOutput renameObject(RenameObjectInput input) throws TosException {
+ return retry(() -> client.renameObject(input));
+ }
+
+ @Override
+ public RestoreObjectOutput restoreObject(RestoreObjectInput input) throws TosException {
+ return retry(() -> client.restoreObject(input));
+ }
+
+ @Override
+ public PutSymlinkOutput putSymlink(PutSymlinkInput input) throws TosException {
+ return retry(() -> client.putSymlink(input));
+ }
+
+ @Override
+ public GetSymlinkOutput getSymlink(GetSymlinkInput input) throws TosException {
+ return retry(() -> client.getSymlink(input));
+ }
+
+ @Override
+ public PreSignedURLOutput preSignedURL(PreSignedURLInput input) throws TosException {
+ return retry(() -> client.preSignedURL(input));
+ }
+
+ @Override
+ public PreSignedPostSignatureOutput preSignedPostSignature(PreSignedPostSignatureInput input)
+ throws TosException {
+ return retry(() -> client.preSignedPostSignature(input));
+ }
+
+ @Override
+ public PreSingedPolicyURLOutput preSingedPolicyURL(PreSingedPolicyURLInput input)
+ throws TosException {
+ return retry(() -> client.preSingedPolicyURL(input));
+ }
+
+ @Override
+ public PreSignedPolicyURLOutput preSignedPolicyURL(PreSignedPolicyURLInput input) throws TosException {
+ return retry(() -> client.preSignedPolicyURL(input));
+ }
+
+ @Override
+ public void changeCredentials(Credentials credentials) {
+ retry(() -> {
+ client.changeCredentials(credentials);
+ return null;
+ });
+ }
+
+ @Override
+ public void changeRegionAndEndpoint(String region, String endpoint) {
+ retry(() -> {
+ client.changeRegionAndEndpoint(region, endpoint);
+ return null;
+ });
+ }
+
+ @Override
+ public void changeTransportConfig(TransportConfig config) {
+ retry(() -> {
+ client.changeTransportConfig(config);
+ return null;
+ });
+ }
+
+ @Override
+ public boolean refreshEndpointRegion(String s, String s1) {
+ return retry(() -> refreshEndpointRegion(s, s1));
+ }
+
+ @Override
+ public boolean refreshCredentials(String s, String s1, String s2) {
+ return retry(() -> refreshCredentials(s, s1, s2));
+ }
+
+ @Override
+ public void close() throws IOException {
+ client.close();
+ }
+
+ private void refresh() throws TosException {
+ Credential credential = provider.credential();
+ if (credentialIsChanged(credential)) {
+ synchronized (this) {
+ if (credentialIsChanged(credential)) {
+ client.changeCredentials(provider);
+ usedCredential = credential;
+ }
+ }
+ }
+ }
+
+ private boolean credentialIsChanged(Credential credential) {
+ return !Objects.equals(credential.getAccessKeyId(), usedCredential.getAccessKeyId())
+ || !Objects.equals(credential.getAccessKeySecret(), usedCredential.getAccessKeySecret())
+ || !Objects.equals(credential.getSecurityToken(), usedCredential.getSecurityToken());
+ }
+
+ private <T> T retry(Callable<T> callable) {
+ int attempt = 0;
+ while (true) {
+ attempt++;
+ try {
+ refresh();
+ return callable.call();
+ } catch (TosException e) {
+ if (attempt >= maxRetryTimes) {
+ LOG.error("Retry exhausted after {} times.", maxRetryTimes);
+ throw e;
+ }
+ if (isRetryableException(e, nonRetryable409ErrorCodes)) {
+ LOG.warn("Retry TOS request in the {} times, error: {}", attempt,
+ Throwables.getRootCause(e).getMessage());
+ try {
+ // last time does not need to sleep
+ Thread.sleep(RetryableUtils.backoff(attempt));
+ } catch (InterruptedException ex) {
+ throw new TosClientException("tos: request interrupted.", ex);
+ }
+ } else {
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ static boolean isRetryableException(TosException e, List<String> nonRetryable409ErrorCodes) {
+ return e.getStatusCode() >= HttpStatus.INTERNAL_SERVER_ERROR
+ || e.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS
+ || e.getCause() instanceof SocketException
+ || e.getCause() instanceof UnknownHostException
+ || e.getCause() instanceof SSLException
+ || e.getCause() instanceof SocketTimeoutException
+ || e.getCause() instanceof InterruptedException
+ || isRetryableTosClientException(e)
+ || isRetryableTosServerException(e, nonRetryable409ErrorCodes);
+ }
+
+ private static boolean isRetryableTosClientException(TosException e) {
+ return e instanceof TosClientException
+ && e.getCause() instanceof IOException
+ && !(e.getCause() instanceof EOFException);
+ }
+
+ private static boolean isRetryableTosServerException(TosException e, List<String> nonRetryable409ErrorCodes) {
+ return e instanceof TosServerException
+ && e.getStatusCode() == HttpStatus.CONFLICT
+ && isRetryableTosConflictException((TosServerException) e, nonRetryable409ErrorCodes);
+ }
+
+ private static boolean isRetryableTosConflictException(TosServerException e, List<String> nonRetryableCodes) {
+ String errorCode = e.getEc();
+ return StringUtils.isEmpty(errorCode) || !nonRetryableCodes.contains(errorCode);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/tos/DelegationClientBuilder.java b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/tos/DelegationClientBuilder.java
new file mode 100644
index 00000000000..d6d7dbae8f9
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/hadoop-tos-core/src/main/java/org/apache/hadoop/fs/tosfs/object/tos/DelegationClientBuilder.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.tos;
+
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.TOSClientConfiguration;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.TosException;
+import org.apache.hadoop.fs.tosfs.shaded.com.volcengine.tos.transport.TransportConfig;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.conf.TosKeys;
+import org.apache.hadoop.fs.tosfs.object.Constants;
+import org.apache.hadoop.fs.tosfs.object.tos.auth.CredentialsProvider;
+import org.apache.hadoop.fs.tosfs.util.ParseUtils;
+import org.apache.hadoop.fs.tosfs.util.TOSClientContextUtils;
+import org.apache.hadoop.util.Preconditions;
... 4534 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org