You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2024/10/31 09:53:17 UTC
(hadoop) 16/30: Integration of TOS: Add TosKeys and ConfKeys.
This is an automated email from the ASF dual-hosted git repository.
jinglun pushed a commit to branch HADOOP-19236
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 9530d6f2e9b42e66b3960bf3c5916f200843db83
Author: lijinglun <li...@bytedance.com>
AuthorDate: Wed Aug 21 17:49:40 2024 +0800
Integration of TOS: Add TosKeys and ConfKeys.
---
.../org/apache/hadoop/fs/tosfs/conf/ConfKeys.java | 17 ++
.../org/apache/hadoop/fs/tosfs/conf/TosKeys.java | 203 +++++++++++++++++++++
2 files changed, 220 insertions(+)
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java
index 8bf122686cd..a059baaf11e 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java
@@ -20,8 +20,18 @@ package org.apache.hadoop.fs.tosfs.conf;
public class ConfKeys {
+ /**
+ * Object storage endpoint to connect to, which should include both region and object domain name.
+ * e.g. 'fs.tos.endpoint'='tos-cn-beijing.volces.com'.
+ */
public static final ArgumentKey FS_TOS_ENDPOINT = new ArgumentKey("fs.%s.endpoint");
+ /**
+ * The region of the object storage, e.g. fs.tos.region. Parsing template "fs.%s.endpoint" to
+ * know the region.
+ */
+ public static final ArgumentKey FS_TOS_REGION = new ArgumentKey("fs.%s.region");
+
/**
* The object storage implementation for the defined scheme. For example, we can delegate the
* scheme 'abc' to TOS (or other object storage),and access the TOS object storage as
@@ -29,4 +39,11 @@ public class ConfKeys {
*/
public static final ArgumentKey FS_OBJECT_STORAGE_IMPL =
new ArgumentKey("fs.objectstorage.%s.impl");
+
+ /**
+ * The batch size of deleting multiple objects per request for the given object storage.
+ * e.g. fs.tos.delete.batch-size
+ */
+ public static final ArgumentKey FS_BATCH_DELETE_SIZE = new ArgumentKey("fs.%s.delete.batch-size");
+ public static final int FS_BATCH_DELETE_SIZE_DEFAULT = 250;
}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/TosKeys.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/TosKeys.java
index ca4e863654c..075f3a2d09e 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/TosKeys.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/TosKeys.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.fs.tosfs.conf;
+import org.apache.hadoop.fs.tosfs.object.ChecksumType;
+import org.apache.hadoop.fs.tosfs.object.tos.TOSErrorCodes;
+
public class TosKeys {
/**
@@ -72,4 +75,204 @@ public class TosKeys {
public static final String[] FS_TOS_CUSTOM_CREDENTIAL_PROVIDER_CLASSES_DEFAULT =
new String[] { "org.apache.hadoop.fs.tosfs.object.tos.auth.EnvironmentCredentialsProvider",
"org.apache.hadoop.fs.tosfs.object.tos.auth.SimpleCredentialsProvider" };
+
+ /**
+ * Set a canned ACL for newly created and copied objects. Value may be 'private', 'public-read',
+ * 'public-read-write', 'authenticated-read', 'bucket-owner-read', 'bucket-owner-full-control',
+ * 'bucket-owner-entrusted'. If set, caller IAM role must have 'tos:PutObjectAcl' permission on
+ * the bucket.
+ */
+ public static final String FS_TOS_ACL_DEFAULT = "fs.tos.acl.default";
+
+ // TOS http client.
+ /**
+ * The maximum number of connections to the TOS service that a client can create.
+ */
+ public static final String FS_TOS_HTTP_MAX_CONNECTIONS = "fs.tos.http.maxConnections";
+ public static final int FS_TOS_HTTP_MAX_CONNECTIONS_DEFAULT = 1024;
+
+ /**
+ * The time that a connection thread can be in idle state, larger than which the thread will be
+ * terminated.
+ */
+ public static final String FS_TOS_HTTP_IDLE_CONNECTION_TIME_MILLS =
+ "fs.tos.http.idleConnectionTimeMills";
+ public static final int FS_TOS_HTTP_IDLE_CONNECTION_TIME_MILLS_DEFAULT = 60000;
+
+ /**
+ * The connect timeout that the tos client tries to connect to the TOS service.
+ */
+ public static final String FS_TOS_HTTP_CONNECT_TIMEOUT_MILLS = "fs.tos.http.connectTimeoutMills";
+ public static final int FS_TOS_HTTP_CONNECT_TIMEOUT_MILLS_DEFAULT = 10000;
+
+ /**
+ * The reading timeout when reading data from tos. Note that it is configured for the tos client,
+ * not proton.
+ */
+ public static final String FS_TOS_HTTP_READ_TIMEOUT_MILLS = "fs.tos.http.readTimeoutMills";
+ public static final int FS_TOS_HTTP_READ_TIMEOUT_MILLS_DEFAULT = 30000;
+
+ /**
+ * The writing timeout when uploading data to tos. Note that it is configured for the tos client,
+ * not proton.
+ */
+ public static final String FS_TOS_HTTP_WRITE_TIMEOUT_MILLS = "fs.tos.http.writeTimeoutMills";
+ public static final int FS_TOS_HTTP_WRITE_TIMEOUT_MILLS_DEFAULT = 30000;
+
+ /**
+ * Enables SSL connections to TOS or not.
+ */
+ public static final String FS_TOS_HTTP_ENABLE_VERIFY_SSL = "fs.tos.http.enableVerifySSL";
+ public static final boolean FS_TOS_HTTP_ENABLE_VERIFY_SSL_DEFAULT = true;
+
+ /**
+ * The timeout (in minutes) of the dns cache used in tos client.
+ */
+ public static final String FS_TOS_HTTP_DNS_CACHE_TIME_MINUTES = "fs.tos.http.dnsCacheTimeMinutes";
+ public static final int FS_TOS_HTTP_DNS_CACHE_TIME_MINUTES_DEFAULT = 0;
+
+ /**
+ * True to create the missed parent dir asynchronously during deleting or renaming a file or dir.
+ */
+ public static final String FS_ASYNC_CREATE_MISSED_PARENT = "fs.tos.missed.parent.dir.async-create";
+ public static final boolean FS_ASYNC_CREATE_MISSED_PARENT_DEFAULT = false;
+
+ /**
+ * Whether enable tos getFileStatus API or not, which returns the object info directly in one RPC
+ * request, otherwise, might need to send three RPC requests to get object info.
+ * For example, there is a key 'a/b/c' exists in TOS, and we want to get object status of 'a/b',
+ * the GetFileStatus('a/b') will return the prefix 'a/b/' as a directory object directly. If this
+ * property is disabled, we need to head('a/b') at first, and then head('a/b/'), and last call
+ * list('a/b/', limit=1) to get object info. Using GetFileStatus API can reduce the RPC call
+ * times.
+ */
+ public static final String FS_TOS_GET_FILE_STATUS_ENABLED = "fs.tos.get-file-status.enabled";
+ public static final boolean FS_TOS_GET_FILE_STATUS_ENABLED_DEFAULT = true;
+
+ /**
+ * Used for directory bucket, whether enable recursive delete capability in TOS server, which will
+ * atomic delete all objects under given dir(inclusive), otherwise the client will list all sub
+ * objects, and then send batch delete request to TOS to delete dir.
+ */
+ public static final String FS_TOS_RMR_SERVER_ENABLED = "fs.tos.rmr.server.enabled";
+ public static final boolean FS_FS_TOS_RMR_SERVER_ENABLED_DEFAULT = false;
+
+ /**
+ * If fs.tos.rmr.client.enabled is true, client will list all objects under the given dir and
+ * delete them by batch. Set value with true will use the recursive delete capability of TOS SDK,
+ * otherwise will delete object one by one via preorder tree walk.
+ */
+ public static final String FS_TOS_RMR_CLIENT_ENABLE = "fs.tos.rmr.client.enabled";
+ public static final boolean FS_TOS_RMR_CLIENT_ENABLE_DEFAULT = true;
+
+ /**
+ * The prefix will be used as the product name in TOS SDK. The final user agent pattern is
+ * '{prefix}/Proton/{proton version}'.
+ * TODO: review it.
+ */
+ public static final String FS_TOS_USER_AGENT_PREFIX = "fs.tos.user.agent.prefix";
+ public static final String FS_TOS_USER_AGENT_PREFIX_DEFAULT = "EMR";
+
+ /**
+ * The key indicates the name of the tos checksum algorithm. Specify the algorithm name to compare
+ * checksums between different storage systems. For example to compare checksums between hdfs and
+ * tos, we need to configure the algorithm name to COMPOSITE-CRC32C.
+ */
+ public static final String FS_TOS_CHECKSUM_ALGORITHM = "fs.tos.checksum-algorithm";
+ public static final String FS_TOS_CHECKSUM_ALGORITHM_DEFAULT = "PROTON-CHECKSUM";
+
+ /**
+ * The key indicates how to retrieve file checksum from tos, error will be thrown if the
+ * configured checksum type is not supported by tos. The supported checksum types are:
+ * CRC32C, CRC64ECMA.
+ */
+ public static final String FS_TOS_CHECKSUM_TYPE = "fs.tos.checksum-type";
+ public static final String FS_TOS_CHECKSUM_TYPE_DEFAULT = ChecksumType.CRC64ECMA.name();
+
+ // TOS common keys.
+ /**
+ * The threshold indicates whether reuse the socket connection to optimize read performance during
+ * closing tos object inputstream of get object. If the remaining bytes is less than max drain
+ * bytes during closing the inputstream, will just skip the bytes instead of closing the socket
+ * connection.
+ */
+ public static final String FS_TOS_MAX_DRAIN_BYTES = "fs.tos.max-drain-bytes";
+ public static final long FS_TOS_MAX_DRAIN_BYTES_DEFAULT = 1024 * 1024L;
+
+ /**
+ * Whether disable the tos http client cache in the current JVM.
+ */
+ public static final String FS_TOS_DISABLE_CLIENT_CACHE = "fs.tos.client.disable.cache";
+ public static final boolean FS_TOS_DISABLE_CLIENT_CACHE_DEFAULT = false;
+
+ /**
+ * The batch size when deleting the objects in batches.
+ */
+ public static final String FS_TOS_DELETE_OBJECTS_COUNT = "fs.tos.batch.delete.objects-count";
+ public static final int FS_TOS_DELETE_OBJECTS_COUNT_DEFAULT = 1000;
+
+ /**
+ * The maximum retry times when deleting objects in batches failed.
+ */
+ public static final String FS_TOS_BATCH_DELETE_MAX_RETRIES = "fs.tos.batch.delete.max-retries";
+ public static final int FS_TOS_BATCH_DELETE_MAX_RETRIES_DEFAULT = 20;
+
+ /**
+ * The codes from TOS deleteMultiObjects response, Proton will resend the batch delete request to
+ * delete the failed keys again if the response only contains these codes, otherwise won't send
+ * request anymore.
+ */
+ public static final String FS_TOS_BATCH_DELETE_RETRY_CODES = "fs.tos.batch.delete.retry-codes";
+ public static final String[] FS_TOS_BATCH_DELETE_RETRY_CODES_DEFAULT =
+ new String[] { "ExceedAccountQPSLimit", "ExceedAccountRateLimit", "ExceedBucketQPSLimit",
+ "ExceedBucketRateLimit", "InternalError", "ServiceUnavailable", "SlowDown",
+ "TooManyRequests" };
+
+ /**
+ * The retry interval (in milliseconds) when deleting objects in batches failed.
+ */
+ public static final String FS_TOS_BATCH_DELETE_RETRY_INTERVAL =
+ "fs.tos.batch.delete.retry.interval";
+ public static final long FS_TOS_BATCH_DELETE_RETRY_INTERVAL_DEFAULT = 1000L;
+
+ /**
+ * The batch size of listing object per request for the given object storage, such as listing a
+ * directory, searching for all objects whose path starts with the directory path, and returning
+ * them as a list.
+ */
+ public static final String FS_TOS_LIST_OBJECTS_COUNT = "fs.tos.list.objects-count";
+ public static final int FS_TOS_LIST_OBJECTS_COUNT_DEFAULT = 1000;
+
+ /**
+ * The maximum retry times of sending request via TOS client, Proton will resend the request if
+ * got retryable exceptions, e.g. SocketException, UnknownHostException, SSLException,
+ * InterruptedException, SocketTimeoutException, or got TOO_MANY_REQUESTS, INTERNAL_SERVER_ERROR
+ * http codes.
+ */
+ public static final String FS_TOS_REQUEST_MAX_RETRY_TIMES = "fs.tos.request.max.retry.times";
+ public static final int FS_TOS_REQUEST_MAX_RETRY_TIMES_DEFAULT = 20;
+
+ /**
+ * The fast-fail error codes means the error cannot be solved by retrying the request. TOS client
+ * won't retry the request if receiving a 409 http status code and if the error code is in the
+ * configured non-retryable error code list.
+ */
+ public static final String FS_TOS_FAST_FAILURE_409_ERROR_CODES =
+ "fs.tos.fast-fail-409-error-codes";
+ public static final String FS_TOS_FAST_FAILURE_409_ERROR_CODES_DEFAULT =
+ TOSErrorCodes.FAST_FAILURE_CONFLICT_ERROR_CODES;
+
+ /**
+ * The maximum retry times of reading object content via TOS client, Proton will resend the
+ * request to create a new input stream if getting unexpected end of stream error during reading
+ * the input stream.
+ */
+ public static final String FS_TOS_MAX_READ_OBJECT_RETRIES = "fs.tos.inputstream.max.retry.times";
+ public static final int FS_TOS_MAX_READ_OBJECT_RETRIES_DEFAULT = 5;
+
+ /**
+ * Enable the crc check when uploading files to tos or not.
+ */
+ public static final String FS_TOS_CRC_CHECK_ENABLED = "fs.tos.crc.check.enable";
+ public static final boolean FS_TOS_CRC_CHECK_ENABLED_DEFAULT = true;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org