You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/05/16 14:18:26 UTC
(opendal) branch main updated: fix(dev/generator): fix service generation after services refactor (#7531)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new ce05b7247 fix(dev/generator): fix service generation after services refactor (#7531)
ce05b7247 is described below
commit ce05b7247d49d35f698661deb2a37367ccef9ed8
Author: Trim21 <i...@trim21.me>
AuthorDate: Sat May 16 22:18:20 2026 +0800
fix(dev/generator): fix service generation after services refactor (#7531)
---
.github/workflows/ci_odev.yml | 20 +
.../java/org/apache/opendal/ServiceConfig.java | 555 +++++++-
bindings/python/Cargo.toml | 4 +
bindings/python/justfile | 2 +-
bindings/python/python/opendal/capability.pyi | 15 +
bindings/python/python/opendal/exceptions.pyi | 6 +-
bindings/python/python/opendal/file.pyi | 6 +-
bindings/python/python/opendal/operator.pyi | 1340 +++++++++++++-----
bindings/python/python/opendal/services.pyi | 7 +-
bindings/python/ruff.toml | 3 +
bindings/python/src/file.rs | 6 +-
bindings/python/src/operator.rs | 24 +-
bindings/python/src/services.rs | 1440 +++++++++++++++-----
bindings/python/tests/test_async_conditional.py | 1 -
bindings/python/tests/test_stat_conditional.py | 8 +-
bindings/python/tests/test_sync_conditional.py | 1 -
bindings/python/tests/test_write_conditional.py | 8 +-
dev/src/generate/java.rs | 6 +
dev/src/generate/mod.rs | 17 +-
dev/src/generate/parser.rs | 54 +-
dev/src/generate/python.j2 | 16 +-
dev/src/generate/python.rs | 1 +
22 files changed, 2725 insertions(+), 815 deletions(-)
diff --git a/.github/workflows/ci_odev.yml b/.github/workflows/ci_odev.yml
index 0bc9ecf01..aa73684ff 100644
--- a/.github/workflows/ci_odev.yml
+++ b/.github/workflows/ci_odev.yml
@@ -60,3 +60,23 @@ jobs:
- name: Cargo Test
working-directory: dev
run: cargo test
+
+ check_generated:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Generate Python bindings code
+ run: cargo run --manifest-path=dev/Cargo.toml -- generate -l python
+
+ - name: Generate Java bindings code
+ run: cargo run --manifest-path=dev/Cargo.toml -- generate -l java
+ - run: cargo fmt --manifest-path bindings/python/Cargo.toml --all
+
+ - name: Check diff
+ run: git diff --exit-code
diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
index 51315855d..52175686d 100644
--- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
+++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
@@ -51,7 +51,7 @@ public interface ServiceConfig {
Map<String, String> configMap();
/**
- * Configuration for service aliyun_drive.
+ * Configuration for service aliyun-drive.
*/
@Builder
@Data
@@ -94,7 +94,7 @@ public interface ServiceConfig {
@Override
public String scheme() {
- return "aliyun_drive";
+ return "aliyun-drive";
}
@Override
@@ -294,6 +294,14 @@ public interface ServiceConfig {
* </ul>
*/
public final String clientSecret;
+ /**
+ * <p>Whether hierarchical namespace (HNS) is enabled for the storage account.
+ * When enabled, recursive deletion can use pagination to avoid timeouts on large directories.</p>
+ * <ul>
+ * <li>default value: <code>false</code></li>
+ * </ul>
+ */
+ public final Boolean enableHns;
/**
* <p>Endpoint of this backend.</p>
*/
@@ -346,6 +354,9 @@ public interface ServiceConfig {
if (clientSecret != null) {
map.put("client_secret", clientSecret);
}
+ if (enableHns != null) {
+ map.put("enable_hns", String.valueOf(enableHns));
+ }
if (endpoint != null) {
map.put("endpoint", endpoint);
}
@@ -513,7 +524,7 @@ public interface ServiceConfig {
}
/**
- * Configuration for service cloudflare_kv.
+ * Configuration for service cloudflare-kv.
*/
@Builder
@Data
@@ -542,7 +553,7 @@ public interface ServiceConfig {
@Override
public String scheme() {
- return "cloudflare_kv";
+ return "cloudflare-kv";
}
@Override
@@ -934,6 +945,94 @@ public interface ServiceConfig {
}
}
+ /**
+ * Configuration for service foyer.
+ */
+ @Builder
+ @Data
+ @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+ class Foyer implements ServiceConfig {
+ /**
+ * <p>Disk cache total capacity in bytes.
+ * Only used when <code>disk_path</code> is set.</p>
+ */
+ public final Long diskCapacity;
+ /**
+ * <p>Individual cache file size in bytes.</p>
+ * <p>Default is 1 MiB.
+ * Only used when <code>disk_path</code> is set.</p>
+ */
+ public final Long diskFileSize;
+ /**
+ * <p>Disk cache directory path.</p>
+ * <p>If set, enables hybrid cache with disk storage. Data will be persisted to
+ * this directory when memory cache is full.</p>
+ */
+ public final String diskPath;
+ /**
+ * <p>Memory capacity in bytes for the cache.</p>
+ */
+ public final Long memory;
+ /**
+ * <p>Name for this cache instance.</p>
+ */
+ public final String name;
+ /**
+ * <p>Recovery mode when starting the cache.</p>
+ * <p>Valid values: "none" (default), "quiet", "strict".</p>
+ * <ul>
+ * <li>"none": Don't recover from disk</li>
+ * <li>"quiet": Recover and skip errors</li>
+ * <li>"strict": Recover and panic on errors</li>
+ * </ul>
+ */
+ public final String recoverMode;
+ /**
+ * <p>Root path of this backend.</p>
+ */
+ public final String root;
+ /**
+ * <p>Number of shards for concurrent access.</p>
+ * <p>Default is 1. Higher values improve concurrency but increase overhead.</p>
+ */
+ public final Long shards;
+
+ @Override
+ public String scheme() {
+ return "foyer";
+ }
+
+ @Override
+ public Map<String, String> configMap() {
+ final HashMap<String, String> map = new HashMap<>();
+ if (diskCapacity != null) {
+ map.put("disk_capacity", String.valueOf(diskCapacity));
+ }
+ if (diskFileSize != null) {
+ map.put("disk_file_size", String.valueOf(diskFileSize));
+ }
+ if (diskPath != null) {
+ map.put("disk_path", diskPath);
+ }
+ if (memory != null) {
+ map.put("memory", String.valueOf(memory));
+ }
+ if (name != null) {
+ map.put("name", name);
+ }
+ if (recoverMode != null) {
+ map.put("recover_mode", recoverMode);
+ }
+ if (root != null) {
+ map.put("root", root);
+ }
+ if (shards != null) {
+ map.put("shards", String.valueOf(shards));
+ }
+ return map;
+ }
+ }
+
/**
* Configuration for service fs.
*/
@@ -1234,6 +1333,98 @@ public interface ServiceConfig {
}
}
+ /**
+ * Configuration for service goosefs.
+ */
+ @Builder
+ @Data
+ @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+ class Goosefs implements ServiceConfig {
+ /**
+ * <p>Authentication type.</p>
+ * <p>Supported values: <code>"nosasl"</code>, <code>"simple"</code>.
+ * Default: <code>"simple"</code> — PLAIN SASL with username.
+ * <code>"nosasl"</code> — skip authentication entirely.</p>
+ */
+ public final String authType;
+ /**
+ * <p>Authentication username.</p>
+ * <p>Used in SIMPLE mode as the login identity.
+ * Default: current OS user (<code>$USER</code> / <code>$USERNAME</code>).</p>
+ */
+ public final String authUsername;
+ /**
+ * <p>Block size in bytes for new files (default: 64 MiB).</p>
+ */
+ public final Long blockSize;
+ /**
+ * <p>Chunk size in bytes for streaming RPCs (default: 1 MiB).</p>
+ */
+ public final Long chunkSize;
+ /**
+ * <p>Master address(es) in <code>host:port</code> format.</p>
+ * <p>For single master: <code>"10.0.0.1:9200"</code>
+ * For HA (comma-separated): <code>"10.0.0.1:9200,10.0.0.2:9200,10.0.0.3:9200"</code></p>
+ * <p>When multiple addresses are provided, the client uses
+ * <code>PollingMasterInquireClient</code> to discover the Primary Master automatically.</p>
+ * <p>Resolution precedence at <code>build()</code> time (highest → lowest), following
+ * <code>goosefs-sdk</code> <code>docs/CLIENT_CONFIGURATION.md</code> §1:</p>
+ * <ol>
+ * <li>This field (when set on the builder / OpenDAL config map)</li>
+ * <li><code>GOOSEFS_MASTER_ADDR</code> environment variable</li>
+ * <li><code>goosefs.master.rpc.addresses</code> / <code>goosefs.master.hostname</code> in
+ * <code>goosefs-site.properties</code></li>
+ * </ol>
+ * <p><code>build()</code> fails with <code>ConfigInvalid</code> only when <strong>none</strong> of the above
+ * supplies a master address.</p>
+ */
+ public final String masterAddr;
+ /**
+ * <p>Root path of this backend.</p>
+ * <p>All operations will happen under this root.
+ * Default to <code>/</code> if not set.</p>
+ */
+ public final String root;
+ /**
+ * <p>Default write type for new files.</p>
+ * <p>Supported values: <code>"must_cache"</code>, <code>"cache_through"</code>, <code>"through"</code>, <code>"async_through"</code>.
+ * Default: <code>"must_cache"</code>.</p>
+ */
+ public final String writeType;
+
+ @Override
+ public String scheme() {
+ return "goosefs";
+ }
+
+ @Override
+ public Map<String, String> configMap() {
+ final HashMap<String, String> map = new HashMap<>();
+ if (authType != null) {
+ map.put("auth_type", authType);
+ }
+ if (authUsername != null) {
+ map.put("auth_username", authUsername);
+ }
+ if (blockSize != null) {
+ map.put("block_size", String.valueOf(blockSize));
+ }
+ if (chunkSize != null) {
+ map.put("chunk_size", String.valueOf(chunkSize));
+ }
+ if (masterAddr != null) {
+ map.put("master_addr", masterAddr);
+ }
+ if (root != null) {
+ map.put("root", root);
+ }
+ if (writeType != null) {
+ map.put("write_type", writeType);
+ }
+ return map;
+ }
+ }
+
/**
* Configuration for service gridfs.
*/
@@ -1290,7 +1481,7 @@ public interface ServiceConfig {
}
/**
- * Configuration for service hdfs_native.
+ * Configuration for service hdfs-native.
*/
@Builder
@Data
@@ -1304,6 +1495,10 @@ public interface ServiceConfig {
* <p>name_node of this backend</p>
*/
public final String nameNode;
+ /**
+ * <p>other options for hdfs client</p>
+ */
+ public final Map<String, String> options;
/**
* <p>work dir of this backend</p>
*/
@@ -1311,7 +1506,7 @@ public interface ServiceConfig {
@Override
public String scheme() {
- return "hdfs_native";
+ return "hdfs-native";
}
@Override
@@ -1323,6 +1518,9 @@ public interface ServiceConfig {
if (nameNode != null) {
map.put("name_node", nameNode);
}
+ if (options != null) {
+ map.putAll(options);
+ }
if (root != null) {
map.put("root", root);
}
@@ -1331,36 +1529,46 @@ public interface ServiceConfig {
}
/**
- * Configuration for service http.
+ * Configuration for service hf.
*/
@Builder
@Data
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
- class Http implements ServiceConfig {
+ class Hf implements ServiceConfig {
/**
- * <p>endpoint of this backend</p>
+ * <p>Endpoint of the Hugging Face Hub.</p>
+ * <p>Default is "https://huggingface.co".</p>
*/
public final String endpoint;
/**
- * <p>password of this backend</p>
+ * <p>Repo id of this backend.</p>
+ * <p>This is required.</p>
*/
- public final String password;
+ public final String repoId;
/**
- * <p>root of this backend</p>
+ * <p>Repo type of this backend. Default is model.</p>
+ * <p>Default is model</p>
*/
- public final String root;
+ public final @NonNull String repoType;
/**
- * <p>token of this backend</p>
+ * <p>Revision of this backend.</p>
+ * <p>Default is main.</p>
*/
- public final String token;
+ public final String revision;
/**
- * <p>username of this backend</p>
+ * <p>Root of this backend. Can be "/path/to/dir".</p>
+ * <p>Default is "/".</p>
*/
- public final String username;
+ public final String root;
+ /**
+ * <p>Token of this backend.</p>
+ * <p>This is optional.</p>
+ */
+ public final String token;
@Override
public String scheme() {
- return "http";
+ return "hf";
}
@Override
@@ -1369,8 +1577,12 @@ public interface ServiceConfig {
if (endpoint != null) {
map.put("endpoint", endpoint);
}
- if (password != null) {
- map.put("password", password);
+ if (repoId != null) {
+ map.put("repo_id", repoId);
+ }
+ map.put("repo_type", repoType);
+ if (revision != null) {
+ map.put("revision", revision);
}
if (root != null) {
map.put("root", root);
@@ -1378,66 +1590,51 @@ public interface ServiceConfig {
if (token != null) {
map.put("token", token);
}
- if (username != null) {
- map.put("username", username);
- }
return map;
}
}
/**
- * Configuration for service huggingface.
+ * Configuration for service http.
*/
@Builder
@Data
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
- class Huggingface implements ServiceConfig {
- /**
- * <p>Repo id of this backend.</p>
- * <p>This is required.</p>
- */
- public final String repoId;
+ class Http implements ServiceConfig {
/**
- * <p>Repo type of this backend. Default is model.</p>
- * <p>Available values:</p>
- * <ul>
- * <li>model</li>
- * <li>dataset</li>
- * </ul>
+ * <p>endpoint of this backend</p>
*/
- public final String repoType;
+ public final String endpoint;
/**
- * <p>Revision of this backend.</p>
- * <p>Default is main.</p>
+ * <p>password of this backend</p>
*/
- public final String revision;
+ public final String password;
/**
- * <p>Root of this backend. Can be "/path/to/dir".</p>
- * <p>Default is "/".</p>
+ * <p>root of this backend</p>
*/
public final String root;
/**
- * <p>Token of this backend.</p>
- * <p>This is optional.</p>
+ * <p>token of this backend</p>
*/
public final String token;
+ /**
+ * <p>username of this backend</p>
+ */
+ public final String username;
@Override
public String scheme() {
- return "huggingface";
+ return "http";
}
@Override
public Map<String, String> configMap() {
final HashMap<String, String> map = new HashMap<>();
- if (repoId != null) {
- map.put("repo_id", repoId);
- }
- if (repoType != null) {
- map.put("repo_type", repoType);
+ if (endpoint != null) {
+ map.put("endpoint", endpoint);
}
- if (revision != null) {
- map.put("revision", revision);
+ if (password != null) {
+ map.put("password", password);
}
if (root != null) {
map.put("root", root);
@@ -1445,6 +1642,9 @@ public interface ServiceConfig {
if (token != null) {
map.put("token", token);
}
+ if (username != null) {
+ map.put("username", username);
+ }
return map;
}
}
@@ -1641,7 +1841,7 @@ public interface ServiceConfig {
* <p>The maximum number of connections allowed.</p>
* <p>default is 10</p>
*/
- public final Integer connectionPoolMaxSize;
+ public final Long connectionPoolMaxSize;
/**
* <p>The default ttl for put operations.</p>
*/
@@ -2117,6 +2317,10 @@ public interface ServiceConfig {
@Data
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
class Opfs implements ServiceConfig {
+ /**
+ * <p>Root directory for this backend.</p>
+ */
+ public final String root;
@Override
public String scheme() {
@@ -2126,6 +2330,9 @@ public interface ServiceConfig {
@Override
public Map<String, String> configMap() {
final HashMap<String, String> map = new HashMap<>();
+ if (root != null) {
+ map.put("root", root);
+ }
return map;
}
}
@@ -2141,7 +2348,7 @@ public interface ServiceConfig {
* <p>Access key id for oss.</p>
* <ul>
* <li>this field if it's <code>is_some</code></li>
- * <li>env value: [<code>ALIBABA_CLOUD_ACCESS_KEY_ID</code>]</li>
+ * <li>env value: <code>ALIBABA_CLOUD_ACCESS_KEY_ID</code></li>
* </ul>
*/
public final String accessKeyId;
@@ -2149,7 +2356,7 @@ public interface ServiceConfig {
* <p>Access key secret for oss.</p>
* <ul>
* <li>this field if it's <code>is_some</code></li>
- * <li>env value: [<code>ALIBABA_CLOUD_ACCESS_KEY_SECRET</code>]</li>
+ * <li>env value: <code>ALIBABA_CLOUD_ACCESS_KEY_SECRET</code></li>
* </ul>
*/
public final String accessKeySecret;
@@ -2187,7 +2394,7 @@ public interface ServiceConfig {
* <p><code>oidc_provider_arn</code> will be loaded from</p>
* <ul>
* <li>this field if it's <code>is_some</code></li>
- * <li>env value: [<code>ALIBABA_CLOUD_OIDC_PROVIDER_ARN</code>]</li>
+ * <li>env value: <code>ALIBABA_CLOUD_OIDC_PROVIDER_ARN</code></li>
* </ul>
*/
public final String oidcProviderArn;
@@ -2195,7 +2402,7 @@ public interface ServiceConfig {
* <p><code>oidc_token_file</code> will be loaded from</p>
* <ul>
* <li>this field if it's <code>is_some</code></li>
- * <li>env value: [<code>ALIBABA_CLOUD_OIDC_TOKEN_FILE</code>]</li>
+ * <li>env value: <code>ALIBABA_CLOUD_OIDC_TOKEN_FILE</code></li>
* </ul>
*/
public final String oidcTokenFile;
@@ -2212,7 +2419,7 @@ public interface ServiceConfig {
* credential to assume role with <code>role_arn</code>.</p>
* <ul>
* <li>this field if it's <code>is_some</code></li>
- * <li>env value: [<code>ALIBABA_CLOUD_ROLE_ARN</code>]</li>
+ * <li>env value: <code>ALIBABA_CLOUD_ROLE_ARN</code></li>
* </ul>
*/
public final String roleArn;
@@ -2228,7 +2435,7 @@ public interface ServiceConfig {
* <p><code>security_token</code> will be loaded from</p>
* <ul>
* <li>this field if it's <code>is_some</code></li>
- * <li>env value: [<code>ALIBABA_CLOUD_SECURITY_TOKEN</code>]</li>
+ * <li>env value: <code>ALIBABA_CLOUD_SECURITY_TOKEN</code></li>
* </ul>
*/
public final String securityToken;
@@ -2244,7 +2451,7 @@ public interface ServiceConfig {
* <p><code>sts_endpoint</code> will be loaded from</p>
* <ul>
* <li>this field if it's <code>is_some</code></li>
- * <li>env value: [<code>ALIBABA_CLOUD_STS_ENDPOINT</code>]</li>
+ * <li>env value: <code>ALIBABA_CLOUD_STS_ENDPOINT</code></li>
* </ul>
*/
public final String stsEndpoint;
@@ -2528,7 +2735,7 @@ public interface ServiceConfig {
* <p>The maximum number of connections allowed.</p>
* <p>default is 10</p>
*/
- public final Integer connectionPoolMaxSize;
+ public final Long connectionPoolMaxSize;
/**
* <p>the number of DBs redis can take is unlimited</p>
* <p>default is db 0</p>
@@ -2613,6 +2820,14 @@ public interface ServiceConfig {
* when credential is not loaded.</p>
*/
public final Boolean allowAnonymous;
+ /**
+ * <p>assume_role_duration_seconds for this backend.</p>
+ */
+ public final Integer assumeRoleDurationSeconds;
+ /**
+ * <p>assume_role_session_tags for this backend.</p>
+ */
+ public final Map<String, String> assumeRoleSessionTags;
/**
* <p>Set maximum batch operations of this backend.</p>
* <p>Some compatible services have a limit on the number of operations in a batch request.
@@ -2633,9 +2848,15 @@ public interface ServiceConfig {
* <p>Available options:</p>
* <ul>
* <li>"crc32c"</li>
+ * <li>"md5"</li>
* </ul>
*/
public final String checksumAlgorithm;
+ /**
+ * <p>Default ACL for new objects.
+ * Note that some s3 services like minio do not support this option.</p>
+ */
+ public final String defaultAcl;
/**
* <p>default storage_class for this backend.</p>
* <p>Available values:</p>
@@ -2674,7 +2895,7 @@ public interface ServiceConfig {
/**
* <p>Disable load credential from ec2 metadata.</p>
* <p>This option is used to disable the default behavior of opendal
- * to load credential from ec2 metadata, a.k.a, IMDSv2</p>
+ * to load credential from ec2 metadata, a.k.a., IMDSv2</p>
*/
public final Boolean disableEc2Metadata;
/**
@@ -2690,7 +2911,7 @@ public interface ServiceConfig {
public final Boolean disableStatWithOverride;
/**
* <p>Disable write with if match so that opendal will not send write request with if match headers.</p>
- * <p>For example, Ceph RADOS S3 doesn't support write with if match.</p>
+ * <p>For example, Ceph RADOS S3 doesn't support write with if matched.</p>
*/
public final Boolean disableWriteWithIfMatch;
/**
@@ -2827,6 +3048,12 @@ public interface ServiceConfig {
if (allowAnonymous != null) {
map.put("allow_anonymous", String.valueOf(allowAnonymous));
}
+ if (assumeRoleDurationSeconds != null) {
+ map.put("assume_role_duration_seconds", String.valueOf(assumeRoleDurationSeconds));
+ }
+ if (assumeRoleSessionTags != null) {
+ map.putAll(assumeRoleSessionTags);
+ }
if (batchMaxOperations != null) {
map.put("batch_max_operations", String.valueOf(batchMaxOperations));
}
@@ -2834,6 +3061,9 @@ public interface ServiceConfig {
if (checksumAlgorithm != null) {
map.put("checksum_algorithm", checksumAlgorithm);
}
+ if (defaultAcl != null) {
+ map.put("default_acl", defaultAcl);
+ }
if (defaultStorageClass != null) {
map.put("default_storage_class", defaultStorageClass);
}
@@ -3236,6 +3466,20 @@ public interface ServiceConfig {
* <p>The root for Swift.</p>
*/
public final String root;
+ /**
+ * <p>The hash algorithm for TempURL signing.</p>
+ * <p>Supported values: <code>sha1</code>, <code>sha256</code>, <code>sha512</code>. Defaults to <code>sha256</code>.
+ * The cluster must have the chosen algorithm in its
+ * <code>tempurl.allowed_digests</code> (check <code>GET /info</code>).</p>
+ */
+ public final String tempUrlHashAlgorithm;
+ /**
+ * <p>The TempURL key for generating presigned URLs.</p>
+ * <p>This corresponds to the <code>X-Account-Meta-Temp-URL-Key</code> or
+ * <code>X-Container-Meta-Temp-URL-Key</code> header value configured on the
+ * Swift account or container.</p>
+ */
+ public final String tempUrlKey;
/**
* <p>The token for Swift.</p>
*/
@@ -3258,6 +3502,12 @@ public interface ServiceConfig {
if (root != null) {
map.put("root", root);
}
+ if (tempUrlHashAlgorithm != null) {
+ map.put("temp_url_hash_algorithm", tempUrlHashAlgorithm);
+ }
+ if (tempUrlKey != null) {
+ map.put("temp_url_key", tempUrlKey);
+ }
if (token != null) {
map.put("token", token);
}
@@ -3265,6 +3515,128 @@ public interface ServiceConfig {
}
}
+ /**
+ * Configuration for service tos.
+ */
+ @Builder
+ @Data
+ @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+ class Tos implements ServiceConfig {
+ /**
+ * <p>access_key_id of this backend.</p>
+ * <ul>
+ * <li>If access_key_id is set, we will take user's input first.</li>
+ * <li>If not, we will try to load it from environment.</li>
+ * </ul>
+ */
+ public final String accessKeyId;
+ /**
+ * <p>Allow anonymous will allow opendal to send request without signing
+ * when credential is not loaded.</p>
+ */
+ public final Boolean allowAnonymous;
+ /**
+ * <p>bucket name of this backend.</p>
+ * <p>required.</p>
+ */
+ public final @NonNull String bucket;
+ /**
+ * <p>Disable config load so that opendal will not load config from
+ * environment.</p>
+ * <p>For examples:</p>
+ * <ul>
+ * <li>envs like <code>TOS_ACCESS_KEY_ID</code></li>
+ * </ul>
+ */
+ public final Boolean disableConfigLoad;
+ /**
+ * <p>Enable bucket versioning for this backend.</p>
+ * <p>If set to true, OpenDAL will support versioned operations like list with
+ * versions, read with version, etc.</p>
+ */
+ public final Boolean enableVersioning;
+ /**
+ * <p>endpoint of this backend.</p>
+ * <p>Endpoint must be full uri, e.g.</p>
+ * <ul>
+ * <li>TOS: <code>https://tos-cn-beijing.volces.com</code></li>
+ * <li>TOS with region: <code>https://tos-{region}.volces.com</code></li>
+ * </ul>
+ * <p>If user inputs endpoint without scheme like "tos-cn-beijing.volces.com", we
+ * will prepend "https://" before it.</p>
+ */
+ public final String endpoint;
+ /**
+ * <p>Region represent the signing region of this endpoint.</p>
+ * <p>Required if endpoint is not provided.</p>
+ * <ul>
+ * <li>If region is set, we will take user's input first.</li>
+ * <li>If not, we will try to load it from environment.</li>
+ * <li>If still not set, default to <code>cn-beijing</code>.</li>
+ * </ul>
+ */
+ public final String region;
+ /**
+ * <p>root of this backend.</p>
+ * <p>All operations will happen under this root.</p>
+ * <p>default to <code>/</code> if not set.</p>
+ */
+ public final String root;
+ /**
+ * <p>secret_access_key of this backend.</p>
+ * <ul>
+ * <li>If secret_access_key is set, we will take user's input first.</li>
+ * <li>If not, we will try to load it from environment.</li>
+ * </ul>
+ */
+ public final String secretAccessKey;
+ /**
+ * <p>security_token of this backend.</p>
+ * <p>This token will expire after sometime, it's recommended to set security_token
+ * by hand.</p>
+ */
+ public final String securityToken;
+
+ @Override
+ public String scheme() {
+ return "tos";
+ }
+
+ @Override
+ public Map<String, String> configMap() {
+ final HashMap<String, String> map = new HashMap<>();
+ if (accessKeyId != null) {
+ map.put("access_key_id", accessKeyId);
+ }
+ if (allowAnonymous != null) {
+ map.put("allow_anonymous", String.valueOf(allowAnonymous));
+ }
+ map.put("bucket", bucket);
+ if (disableConfigLoad != null) {
+ map.put("disable_config_load", String.valueOf(disableConfigLoad));
+ }
+ if (enableVersioning != null) {
+ map.put("enable_versioning", String.valueOf(enableVersioning));
+ }
+ if (endpoint != null) {
+ map.put("endpoint", endpoint);
+ }
+ if (region != null) {
+ map.put("region", region);
+ }
+ if (root != null) {
+ map.put("root", root);
+ }
+ if (secretAccessKey != null) {
+ map.put("secret_access_key", secretAccessKey);
+ }
+ if (securityToken != null) {
+ map.put("security_token", securityToken);
+ }
+ return map;
+ }
+ }
+
/**
* Configuration for service upyun.
*/
@@ -3313,7 +3685,7 @@ public interface ServiceConfig {
}
/**
- * Configuration for service vercel_artifacts.
+ * Configuration for service vercel-artifacts.
*/
@Builder
@Data
@@ -3341,7 +3713,7 @@ public interface ServiceConfig {
@Override
public String scheme() {
- return "vercel_artifacts";
+ return "vercel-artifacts";
}
@Override
@@ -3364,7 +3736,7 @@ public interface ServiceConfig {
}
/**
- * Configuration for service vercel_blob.
+ * Configuration for service vercel-blob.
*/
@Builder
@Data
@@ -3382,7 +3754,7 @@ public interface ServiceConfig {
@Override
public String scheme() {
- return "vercel_blob";
+ return "vercel-blob";
}
@Override
@@ -3409,6 +3781,24 @@ public interface ServiceConfig {
* <p>WebDAV Service doesn't support copy.</p>
*/
public final Boolean disableCopy;
+ /**
+ * <p>Disable automatic parent directory creation before write operations.</p>
+ * <p>By default, OpenDAL creates parent directories using MKCOL before writing files.
+ * This requires PROPFIND support to check directory existence.</p>
+ * <p>Some WebDAV-compatible servers (e.g., bazel-remote) don't support PROPFIND
+ * or don't require explicit directory creation. Enable this option to skip
+ * the MKCOL calls and write files directly.</p>
+ * <p>Default: false</p>
+ */
+ public final Boolean disableCreateDir;
+ /**
+ * <p>Enable user metadata support via WebDAV PROPPATCH.</p>
+ * <p>This feature requires the WebDAV server to support RFC4918 PROPPATCH method.
+ * Not all WebDAV servers support this (e.g., nginx's basic WebDAV module doesn't).
+ * Only enable this if your server supports PROPPATCH (e.g., Apache mod_dav, Nextcloud).</p>
+ * <p>Default: false</p>
+ */
+ public final Boolean enableUserMetadata;
/**
* <p>endpoint of this backend</p>
*/
@@ -3425,6 +3815,21 @@ public interface ServiceConfig {
* <p>token of this backend</p>
*/
public final String token;
+ /**
+ * <p>The XML namespace prefix for user metadata properties.</p>
+ * <p>This prefix is used in PROPPATCH/PROPFIND XML requests.
+ * Different servers may require different prefixes.</p>
+ * <p>Default: "opendal"</p>
+ */
+ public final String userMetadataPrefix;
+ /**
+ * <p>The XML namespace URI for user metadata properties.</p>
+ * <p>This URI uniquely identifies the namespace for custom properties.
+ * Different servers may require different namespace URIs.
+ * For example, Nextcloud might work better with its own namespace.</p>
+ * <p>Default: <code>https://opendal.apache.org/ns</code></p>
+ */
+ public final String userMetadataUri;
/**
* <p>username of this backend</p>
*/
@@ -3441,6 +3846,12 @@ public interface ServiceConfig {
if (disableCopy != null) {
map.put("disable_copy", String.valueOf(disableCopy));
}
+ if (disableCreateDir != null) {
+ map.put("disable_create_dir", String.valueOf(disableCreateDir));
+ }
+ if (enableUserMetadata != null) {
+ map.put("enable_user_metadata", String.valueOf(enableUserMetadata));
+ }
if (endpoint != null) {
map.put("endpoint", endpoint);
}
@@ -3453,6 +3864,12 @@ public interface ServiceConfig {
if (token != null) {
map.put("token", token);
}
+ if (userMetadataPrefix != null) {
+ map.put("user_metadata_prefix", userMetadataPrefix);
+ }
+ if (userMetadataUri != null) {
+ map.put("user_metadata_uri", userMetadataUri);
+ }
if (username != null) {
map.put("username", username);
}
@@ -3523,7 +3940,7 @@ public interface ServiceConfig {
}
/**
- * Configuration for service yandex_disk.
+ * Configuration for service yandex-disk.
*/
@Builder
@Data
@@ -3541,7 +3958,7 @@ public interface ServiceConfig {
@Override
public String scheme() {
- return "yandex_disk";
+ return "yandex-disk";
}
@Override
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
index e4b811422..603635f90 100644
--- a/bindings/python/Cargo.toml
+++ b/bindings/python/Cargo.toml
@@ -79,6 +79,8 @@ services-all = [
'services-sqlite',
'services-swift',
'services-upyun',
+ 'services-foyer',
+ 'services-tos',
'services-vercel-artifacts',
'services-yandex-disk',
]
@@ -110,6 +112,7 @@ services-etcd = [
services-foundationdb = [
"opendal/services-foundationdb",
] # FIXME EXCLUDED: this service needs preinstalled fdb library
+services-foyer = ["opendal/services-foyer"]
services-fs = ["opendal/services-fs"]
services-ftp = ["opendal/services-ftp"]
services-gcs = ["opendal/services-gcs"]
@@ -170,6 +173,7 @@ services-swift = ["opendal/services-swift"]
services-tikv = [
"opendal/services-tikv",
] # FIXME EXCLUDED: this service needs protoc
+services-tos = ["opendal/services-tos"]
services-upyun = ["opendal/services-upyun"]
services-vercel-artifacts = ["opendal/services-vercel-artifacts"]
services-vercel-blob = [
diff --git a/bindings/python/justfile b/bindings/python/justfile
index 6181a1ea6..8338ff831 100644
--- a/bindings/python/justfile
+++ b/bindings/python/justfile
@@ -22,7 +22,7 @@ default: help
# Configuration & Variables
# ==============================================================================
-set ignore-comments := true
+set ignore-comments
# ==============================================================================
# Setup & Maintenance
diff --git a/bindings/python/python/opendal/capability.pyi b/bindings/python/python/opendal/capability.pyi
index 46f82d371..83f741da0 100644
--- a/bindings/python/python/opendal/capability.pyi
+++ b/bindings/python/python/opendal/capability.pyi
@@ -44,6 +44,15 @@ class Capability:
def stat_with_if_none_match(self) -> builtins.bool:
r"""If operator supports stat with if none match."""
@property
+ def stat_with_if_modified_since(self) -> builtins.bool:
+ r"""If operator supports stat with if modified since."""
+ @property
+ def stat_with_if_unmodified_since(self) -> builtins.bool:
+ r"""If operator supports stat with if unmodified since."""
+ @property
+ def stat_with_version(self) -> builtins.bool:
+ r"""If operator supports stat with version."""
+ @property
def read(self) -> builtins.bool:
r"""If the operator supports read operations."""
@property
@@ -134,6 +143,12 @@ class Capability:
def delete(self) -> builtins.bool:
r"""If operator supports delete."""
@property
+ def delete_with_version(self) -> builtins.bool:
+ r"""If delete operations with version are supported."""
+ @property
+ def delete_with_recursive(self) -> builtins.bool:
+ r"""If recursive delete operations are supported."""
+ @property
def copy(self) -> builtins.bool:
r"""If operator supports copy."""
@property
diff --git a/bindings/python/python/opendal/exceptions.pyi b/bindings/python/python/opendal/exceptions.pyi
index 806ebfba0..67d8c59ec 100644
--- a/bindings/python/python/opendal/exceptions.pyi
+++ b/bindings/python/python/opendal/exceptions.pyi
@@ -47,12 +47,12 @@ class NotFound(builtins.Exception):
class PermissionDenied(builtins.Exception):
r"""Permission denied."""
-class RateLimited(builtins.Exception):
- r"""Rate limited."""
-
class RangeNotSatisfied(builtins.Exception):
r"""Range not satisfied."""
+class RateLimited(builtins.Exception):
+ r"""Rate limited."""
+
class Unexpected(builtins.Exception):
r"""Unexpected errors."""
diff --git a/bindings/python/python/opendal/file.pyi b/bindings/python/python/opendal/file.pyi
index 7c2446df1..b0d535132 100644
--- a/bindings/python/python/opendal/file.pyi
+++ b/bindings/python/python/opendal/file.pyi
@@ -23,8 +23,6 @@ import collections.abc
import types
import typing
-import typing_extensions
-
@typing.final
class AsyncFile:
r"""
@@ -118,7 +116,7 @@ class AsyncFile:
-----
A closed file cannot be used for further I/O operations.
"""
- def __aenter__(self) -> typing_extensions.Self: ...
+ def __aenter__(self) -> typing.Self: ...
def __aexit__(
self,
exc_type: type[builtins.BaseException] | None,
@@ -281,7 +279,7 @@ class File:
-----
A closed file cannot be used for further I/O operations.
"""
- def __enter__(self, slf: File) -> File: ...
+ def __enter__(self) -> File: ...
def __exit__(
self,
exc_type: type[builtins.BaseException] | None,
diff --git a/bindings/python/python/opendal/operator.pyi b/bindings/python/python/opendal/operator.pyi
index 7136720b0..2a35b53d8 100644
--- a/bindings/python/python/opendal/operator.pyi
+++ b/bindings/python/python/opendal/operator.pyi
@@ -25,8 +25,6 @@ import os
import pathlib
import typing
-import typing_extensions
-
import opendal.file
import opendal.services
import opendal.types
@@ -99,8 +97,8 @@ class AsyncOperator:
size: builtins.int | None = None,
if_match: builtins.str | None = None,
if_none_match: builtins.str | None = None,
- if_modified_since: datetime.datetime = None,
- if_unmodified_since: datetime.datetime = None,
+ if_modified_since: datetime.datetime | None = None,
+ if_unmodified_since: datetime.datetime | None = None,
content_type: builtins.str | None = None,
cache_control: builtins.str | None = None,
content_disposition: builtins.str | None = None,
@@ -210,8 +208,8 @@ class AsyncOperator:
version: builtins.str | None = None,
if_match: builtins.str | None = None,
if_none_match: builtins.str | None = None,
- if_modified_since: datetime.datetime = None,
- if_unmodified_since: datetime.datetime = None,
+ if_modified_since: datetime.datetime | None = None,
+ if_unmodified_since: datetime.datetime | None = None,
content_type: builtins.str | None = None,
cache_control: builtins.str | None = None,
content_disposition: builtins.str | None = None,
@@ -337,7 +335,11 @@ class AsyncOperator:
An awaitable that completes when the directory is created.
"""
def delete(
- self, path: builtins.str | os.PathLike | pathlib.Path
+ self,
+ path: builtins.str | os.PathLike | pathlib.Path,
+ *,
+ version: builtins.str | None = None,
+ recursive: builtins.bool | None = None,
) -> collections.abc.Awaitable[None]:
r"""
Delete a file at the given path.
@@ -355,6 +357,10 @@ class AsyncOperator:
-------
coroutine
An awaitable that completes when the file is deleted.
+ version : str, optional
+ The version of the file to delete. Only supported on version-aware backends.
+ recursive : bool, optional
+ If True, delete the path recursively. Only supported on backends that support recursive delete.
"""
def exists(
self, path: builtins.str | os.PathLike | pathlib.Path
@@ -410,14 +416,14 @@ class AsyncOperator:
path: builtins.str | os.PathLike | pathlib.Path,
expire_second: builtins.int,
*,
- version: builtins.str | None = ...,
- if_match: builtins.str | None = ...,
- if_none_match: builtins.str | None = ...,
- if_modified_since: datetime.datetime = ...,
- if_unmodified_since: datetime.datetime = ...,
- content_type: builtins.str | None = ...,
- cache_control: builtins.str | None = ...,
- content_disposition: builtins.str | None = ...,
+ version: builtins.str | None = None,
+ if_match: builtins.str | None = None,
+ if_none_match: builtins.str | None = None,
+ if_modified_since: datetime.datetime | None = None,
+ if_unmodified_since: datetime.datetime | None = None,
+ content_type: builtins.str | None = None,
+ cache_control: builtins.str | None = None,
+ content_disposition: builtins.str | None = None,
) -> collections.abc.Awaitable[opendal.types.PresignedRequest]:
r"""
Create a presigned request for a stat operation.
@@ -455,14 +461,14 @@ class AsyncOperator:
path: builtins.str | os.PathLike | pathlib.Path,
expire_second: builtins.int,
*,
- version: builtins.str | None = ...,
- if_match: builtins.str | None = ...,
- if_none_match: builtins.str | None = ...,
- if_modified_since: datetime.datetime = ...,
- if_unmodified_since: datetime.datetime = ...,
- content_type: builtins.str | None = ...,
- cache_control: builtins.str | None = ...,
- content_disposition: builtins.str | None = ...,
+ version: builtins.str | None = None,
+ if_match: builtins.str | None = None,
+ if_none_match: builtins.str | None = None,
+ if_modified_since: datetime.datetime | None = None,
+ if_unmodified_since: datetime.datetime | None = None,
+ content_type: builtins.str | None = None,
+ cache_control: builtins.str | None = None,
+ content_disposition: builtins.str | None = None,
) -> collections.abc.Awaitable[opendal.types.PresignedRequest]:
r"""
Create a presigned request for a read operation.
@@ -500,14 +506,14 @@ class AsyncOperator:
path: builtins.str | os.PathLike | pathlib.Path,
expire_second: builtins.int,
*,
- content_type: builtins.str | None = ...,
- content_disposition: builtins.str | None = ...,
- content_encoding: builtins.str | None = ...,
- cache_control: builtins.str | None = ...,
- if_match: builtins.str | None = ...,
- if_none_match: builtins.str | None = ...,
- if_not_exists: builtins.bool | None = ...,
- user_metadata: typing.Mapping[builtins.str, builtins.str] | None = ...,
+ content_type: builtins.str | None = None,
+ content_disposition: builtins.str | None = None,
+ content_encoding: builtins.str | None = None,
+ cache_control: builtins.str | None = None,
+ if_match: builtins.str | None = None,
+ if_none_match: builtins.str | None = None,
+ if_not_exists: builtins.bool | None = None,
+ user_metadata: typing.Mapping[builtins.str, builtins.str] | None = None,
) -> collections.abc.Awaitable[opendal.types.PresignedRequest]:
r"""
Create a presigned request for a write operation.
@@ -545,7 +551,7 @@ class AsyncOperator:
path: builtins.str | os.PathLike | pathlib.Path,
expire_second: builtins.int,
*,
- version: builtins.str | None = ...,
+ version: builtins.str | None = None,
) -> collections.abc.Awaitable[opendal.types.PresignedRequest]:
r"""
Create a presigned request for a delete operation.
@@ -584,15 +590,12 @@ class AsyncOperator:
"""
@typing.overload
def __new__(
- cls,
- scheme: builtins.str,
- /,
- **kwargs: builtins.str,
- ) -> typing_extensions.Self: ...
+ cls, scheme: builtins.str, /, **kwargs: builtins.str
+ ) -> typing.Self: ...
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.AliyunDrive | typing.Literal["aliyun-drive"],
+ scheme: typing.Literal[opendal.services.Scheme.AliyunDrive, "aliyun-drive"],
/,
*,
access_token: builtins.str = ...,
@@ -601,7 +604,7 @@ class AsyncOperator:
drive_type: builtins.str,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `aliyun-drive` service.
@@ -641,12 +644,12 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Alluxio | typing.Literal["alluxio"],
+ scheme: typing.Literal[opendal.services.Scheme.Alluxio, "alluxio"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `alluxio` service.
@@ -669,7 +672,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Azblob | typing.Literal["azblob"],
+ scheme: typing.Literal[opendal.services.Scheme.Azblob, "azblob"],
/,
*,
account_key: builtins.str = ...,
@@ -682,7 +685,7 @@ class AsyncOperator:
endpoint: builtins.str = ...,
root: builtins.str = ...,
sas_token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `azblob` service.
@@ -723,7 +726,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Azdls | typing.Literal["azdls"],
+ scheme: typing.Literal[opendal.services.Scheme.Azdls, "azdls"],
/,
*,
account_key: builtins.str = ...,
@@ -731,12 +734,13 @@ class AsyncOperator:
authority_host: builtins.str = ...,
client_id: builtins.str = ...,
client_secret: builtins.str = ...,
+ enable_hns: builtins.bool = ...,
endpoint: builtins.str = ...,
filesystem: builtins.str,
root: builtins.str = ...,
sas_token: builtins.str = ...,
tenant_id: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `azdls` service.
@@ -759,6 +763,12 @@ class AsyncOperator:
client_secret The client secret of the service
principal.
- required for client_credentials authentication
+ enable_hns : builtins.bool, optional
+ Whether hierarchical namespace (HNS) is enabled for
+ the storage account.
+ When enabled, recursive deletion can use pagination
+ to avoid timeouts on large directories.
+ - default value: `false`
endpoint : builtins.str, optional
Endpoint of this backend.
filesystem : builtins.str
@@ -780,7 +790,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Azfile | typing.Literal["azfile"],
+ scheme: typing.Literal[opendal.services.Scheme.Azfile, "azfile"],
/,
*,
account_key: builtins.str = ...,
@@ -789,7 +799,7 @@ class AsyncOperator:
root: builtins.str = ...,
sas_token: builtins.str = ...,
share_name: builtins.str,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `azfile` service.
@@ -816,7 +826,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.B2 | typing.Literal["b2"],
+ scheme: typing.Literal[opendal.services.Scheme.B2, "b2"],
/,
*,
application_key: builtins.str = ...,
@@ -824,7 +834,7 @@ class AsyncOperator:
bucket: builtins.str,
bucket_id: builtins.str,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `b2` service.
@@ -858,11 +868,11 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Cacache | typing.Literal["cacache"],
+ scheme: typing.Literal[opendal.services.Scheme.Cacache, "cacache"],
/,
*,
datadir: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `cacache` service.
@@ -879,7 +889,45 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Cos | typing.Literal["cos"],
+ scheme: typing.Literal[opendal.services.Scheme.CloudflareKv, "cloudflare-kv"],
+ /,
+ *,
+ account_id: builtins.str = ...,
+ api_token: builtins.str = ...,
+ default_ttl: typing.Any = ...,
+ namespace_id: builtins.str = ...,
+ root: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `AsyncOperator` for `cloudflare-kv` service.
+
+ Parameters
+ ----------
+ account_id : builtins.str, optional
+ The account ID used to authenticate with CloudFlare.
+ Used as URI path parameter.
+ api_token : builtins.str, optional
+ The token used to authenticate with CloudFlare.
+ default_ttl : typing.Any, optional
+ The default ttl for write operations..
+ a human readable duration string see
+ https://docs.rs/humantime/latest/humantime/fn.parse_duration.html
+ for more details
+ namespace_id : builtins.str, optional
+ The namespace ID.
+ Used as URI path parameter.
+ root : builtins.str, optional
+ Root within this backend.
+
+ Returns
+ -------
+ AsyncOperator
+ The new `AsyncOperator` for `cloudflare-kv` service
+ """
+ @typing.overload
+ def __new__(
+ cls,
+ scheme: typing.Literal[opendal.services.Scheme.Cos, "cos"],
/,
*,
bucket: builtins.str = ...,
@@ -889,7 +937,7 @@ class AsyncOperator:
root: builtins.str = ...,
secret_id: builtins.str = ...,
secret_key: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `cos` service.
@@ -919,11 +967,11 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Dashmap | typing.Literal["dashmap"],
+ scheme: typing.Literal[opendal.services.Scheme.Dashmap, "dashmap"],
/,
*,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `dashmap` service.
@@ -940,7 +988,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Dropbox | typing.Literal["dropbox"],
+ scheme: typing.Literal[opendal.services.Scheme.Dropbox, "dropbox"],
/,
*,
access_token: builtins.str = ...,
@@ -948,7 +996,7 @@ class AsyncOperator:
client_secret: builtins.str = ...,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `dropbox` service.
@@ -973,12 +1021,67 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Fs | typing.Literal["fs"],
+ scheme: typing.Literal[opendal.services.Scheme.Foyer, "foyer"],
+ /,
+ *,
+ disk_capacity: builtins.int = ...,
+ disk_file_size: builtins.int = ...,
+ disk_path: builtins.str = ...,
+ memory: builtins.int = ...,
+ name: builtins.str = ...,
+ recover_mode: builtins.str = ...,
+ root: builtins.str = ...,
+ shards: builtins.int = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `AsyncOperator` for `foyer` service.
+
+ Parameters
+ ----------
+ disk_capacity : builtins.int, optional
+ Disk cache total capacity in bytes.
+ Only used when `disk_path` is set.
+ disk_file_size : builtins.int, optional
+ Individual cache file size in bytes.
+ Default is 1 MiB.
+ Only used when `disk_path` is set.
+ disk_path : builtins.str, optional
+ Disk cache directory path.
+ If set, enables hybrid cache with disk storage.
+ Data will be persisted to this directory when memory
+ cache is full.
+ memory : builtins.int, optional
+ Memory capacity in bytes for the cache.
+ name : builtins.str, optional
+ Name for this cache instance.
+ recover_mode : builtins.str, optional
+ Recovery mode when starting the cache.
+ Valid values: "none" (default), "quiet", "strict".
+ - "none": Don't recover from disk - "quiet": Recover
+ and skip errors - "strict": Recover and panic on
+ errors
+ root : builtins.str, optional
+ Root path of this backend.
+ shards : builtins.int, optional
+ Number of shards for concurrent access.
+ Default is 1.
+ Higher values improve concurrency but increase
+ overhead.
+
+ Returns
+ -------
+ AsyncOperator
+ The new `AsyncOperator` for `foyer` service
+ """
+ @typing.overload
+ def __new__(
+ cls,
+ scheme: typing.Literal[opendal.services.Scheme.Fs, "fs"],
/,
*,
atomic_write_dir: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `fs` service.
@@ -997,14 +1100,14 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Ftp | typing.Literal["ftp"],
+ scheme: typing.Literal[opendal.services.Scheme.Ftp, "ftp"],
/,
*,
endpoint: builtins.str = ...,
password: builtins.str = ...,
root: builtins.str = ...,
user: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `ftp` service.
@@ -1027,7 +1130,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Gcs | typing.Literal["gcs"],
+ scheme: typing.Literal[opendal.services.Scheme.Gcs, "gcs"],
/,
*,
allow_anonymous: builtins.bool = ...,
@@ -1043,7 +1146,7 @@ class AsyncOperator:
scope: builtins.str = ...,
service_account: builtins.str = ...,
token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `gcs` service.
@@ -1091,7 +1194,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Gdrive | typing.Literal["gdrive"],
+ scheme: typing.Literal[opendal.services.Scheme.Gdrive, "gdrive"],
/,
*,
access_token: builtins.str = ...,
@@ -1099,7 +1202,7 @@ class AsyncOperator:
client_secret: builtins.str = ...,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `gdrive` service.
@@ -1124,14 +1227,14 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Ghac | typing.Literal["ghac"],
+ scheme: typing.Literal[opendal.services.Scheme.Ghac, "ghac"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
runtime_token: builtins.str = ...,
version: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `ghac` service.
@@ -1154,7 +1257,74 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Gridfs | typing.Literal["gridfs"],
+ scheme: typing.Literal[opendal.services.Scheme.Goosefs, "goosefs"],
+ /,
+ *,
+ auth_type: builtins.str = ...,
+ auth_username: builtins.str = ...,
+ block_size: builtins.int = ...,
+ chunk_size: builtins.int = ...,
+ master_addr: builtins.str = ...,
+ root: builtins.str = ...,
+ write_type: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `AsyncOperator` for `goosefs` service.
+
+ Parameters
+ ----------
+ auth_type : builtins.str, optional
+ Authentication type.
+ Supported values: `"nosasl"`, `"simple"`.
+ Default: `"simple"` — PLAIN SASL with usernam
+ e. `"nosasl"` — skip authentication entirely.
+ auth_username : builtins.str, optional
+ Authentication username.
+ Used in SIMPLE mode as the login identity.
+ Default: current OS user (`$USER` / `$USERNAME`).
+ block_size : builtins.int, optional
+ Block size in bytes for new files (default: 64 MiB).
+ chunk_size : builtins.int, optional
+ Chunk size in bytes for streaming RPCs (default: 1
+ MiB).
+ master_addr : builtins.str, optional
+ Master address(es) in `host:port` format.
+ For single master: `"10.0.0.1:9200"` For HA
+ (comma-separated):
+ `"10.0.0.1:9200,10.0.0.2:9200,10.0.0.3:9200"` When
+ multiple addresses are provided, the client uses
+ `PollingMasterInquireClient` to discover the Primary
+ Master automatically.
+ Resolution precedence at `build()` time (highest →
+ lowest), following `goosefs-sdk`
+ `docs/CLIENT_CONFIGURATION.md` §1:
+ 1. This field (when set on the builder / OpenDAL
+ config map)
+ 2. `GOOSEFS_MASTER_ADDR` environment variable
+ 3. `goosefs.master.rpc.addresses` /
+ `goosefs.master.hostname` in
+ `goosefs-site.properties` `build()` fails with
+ `ConfigInvalid` only when **none** of the above
+ supplies a master address.
+ root : builtins.str, optional
+ Root path of this backend.
+ All operations will happen under this root.
+ Default to `/` if not set.
+ write_type : builtins.str, optional
+ Default write type for new files.
+ Supported values: `"must_cache"`, `"cache_through"`,
+ `"through"`, `"async_through"`.
+ Default: `"must_cache"`.
+
+ Returns
+ -------
+ AsyncOperator
+ The new `AsyncOperator` for `goosefs` service
+ """
+ @typing.overload
+ def __new__(
+ cls,
+ scheme: typing.Literal[opendal.services.Scheme.Gridfs, "gridfs"],
/,
*,
bucket: builtins.str = ...,
@@ -1162,7 +1332,7 @@ class AsyncOperator:
connection_string: builtins.str = ...,
database: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `gridfs` service.
@@ -1191,13 +1361,14 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.HdfsNative | typing.Literal["hdfs-native"],
+ scheme: typing.Literal[opendal.services.Scheme.HdfsNative, "hdfs-native"],
/,
*,
enable_append: builtins.bool = ...,
name_node: builtins.str = ...,
+ options: builtins.dict = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `hdfs-native` service.
@@ -1207,6 +1378,8 @@ class AsyncOperator:
enable the append capacity
name_node : builtins.str, optional
name_node of this backend
+ options : builtins.dict, optional
+ other options for hdfs client
root : builtins.str, optional
work dir of this backend
@@ -1218,85 +1391,89 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Http | typing.Literal["http"],
+ scheme: typing.Literal[opendal.services.Scheme.Hf, "hf"],
/,
*,
endpoint: builtins.str = ...,
- password: builtins.str = ...,
+ repo_id: builtins.str = ...,
+ repo_type: builtins.str,
+ revision: builtins.str = ...,
root: builtins.str = ...,
token: builtins.str = ...,
- username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
- Create a new `AsyncOperator` for `http` service.
+ Create a new `AsyncOperator` for `hf` service.
Parameters
----------
endpoint : builtins.str, optional
- endpoint of this backend
- password : builtins.str, optional
- password of this backend
+ Endpoint of the Hugging Face Hub.
+ Default is "https://huggingface.co".
+ repo_id : builtins.str, optional
+ Repo id of this backend.
+ This is required.
+ repo_type : builtins.str
+ Repo type of this backend.
+ Default is model.
+ Default is model
+ revision : builtins.str, optional
+ Revision of this backend.
+ Default is main.
root : builtins.str, optional
- root of this backend
+ Root of this backend.
+ Can be "/path/to/dir".
+ Default is "/".
token : builtins.str, optional
- token of this backend
- username : builtins.str, optional
- username of this backend
+ Token of this backend.
+ This is optional.
Returns
-------
AsyncOperator
- The new `AsyncOperator` for `http` service
+ The new `AsyncOperator` for `hf` service
"""
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Hf | typing.Literal["hf"],
+ scheme: typing.Literal[opendal.services.Scheme.Http, "http"],
/,
*,
- repo_id: builtins.str = ...,
- repo_type: builtins.str = ...,
- revision: builtins.str = ...,
+ endpoint: builtins.str = ...,
+ password: builtins.str = ...,
root: builtins.str = ...,
token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ username: builtins.str = ...,
+ ) -> typing.Self:
r"""
- Create a new `AsyncOperator` for `hf` (Hugging Face) service.
+ Create a new `AsyncOperator` for `http` service.
Parameters
----------
- repo_id : builtins.str, optional
- Repo id of this backend.
- This is required.
- repo_type : builtins.str, optional
- Repo type of this backend.
- Default is model.
- Available values: - model - dataset
- revision : builtins.str, optional
- Revision of this backend.
- Default is main.
+ endpoint : builtins.str, optional
+ endpoint of this backend
+ password : builtins.str, optional
+ password of this backend
root : builtins.str, optional
- Root of this backend.
- Can be "/path/to/dir".
- Default is "/".
+ root of this backend
token : builtins.str, optional
- Token of this backend.
- This is optional.
+ token of this backend
+ username : builtins.str, optional
+ username of this backend
Returns
-------
AsyncOperator
- The new `AsyncOperator` for `hf` (Hugging Face) service
+ The new `AsyncOperator` for `http` service
"""
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Ipfs | typing.Literal["ipfs"],
+ scheme: typing.Literal[opendal.services.Scheme.Ipfs, "ipfs"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `ipfs` service.
@@ -1315,12 +1492,12 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Ipmfs | typing.Literal["ipmfs"],
+ scheme: typing.Literal[opendal.services.Scheme.Ipmfs, "ipmfs"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `ipmfs` service.
@@ -1339,14 +1516,14 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Koofr | typing.Literal["koofr"],
+ scheme: typing.Literal[opendal.services.Scheme.Koofr, "koofr"],
/,
*,
email: builtins.str,
endpoint: builtins.str,
password: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `koofr` service.
@@ -1371,7 +1548,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Memcached | typing.Literal["memcached"],
+ scheme: typing.Literal[opendal.services.Scheme.Memcached, "memcached"],
/,
*,
connection_pool_max_size: builtins.int = ...,
@@ -1380,7 +1557,7 @@ class AsyncOperator:
password: builtins.str = ...,
root: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `memcached` service.
@@ -1413,11 +1590,11 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Memory | typing.Literal["memory"],
+ scheme: typing.Literal[opendal.services.Scheme.Memory, "memory"],
/,
*,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `memory` service.
@@ -1434,14 +1611,14 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.MiniMoka | typing.Literal["mini-moka"],
+ scheme: typing.Literal[opendal.services.Scheme.MiniMoka, "mini-moka"],
/,
*,
max_capacity: builtins.int = ...,
root: builtins.str = ...,
time_to_idle: builtins.str = ...,
time_to_live: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `mini-moka` service.
@@ -1470,7 +1647,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Moka | typing.Literal["moka"],
+ scheme: typing.Literal[opendal.services.Scheme.Moka, "moka"],
/,
*,
max_capacity: builtins.int = ...,
@@ -1478,7 +1655,7 @@ class AsyncOperator:
root: builtins.str = ...,
time_to_idle: builtins.str = ...,
time_to_live: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `moka` service.
@@ -1509,7 +1686,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Mongodb | typing.Literal["mongodb"],
+ scheme: typing.Literal[opendal.services.Scheme.Mongodb, "mongodb"],
/,
*,
collection: builtins.str = ...,
@@ -1518,7 +1695,7 @@ class AsyncOperator:
key_field: builtins.str = ...,
root: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `mongodb` service.
@@ -1545,7 +1722,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Mysql | typing.Literal["mysql"],
+ scheme: typing.Literal[opendal.services.Scheme.Mysql, "mysql"],
/,
*,
connection_string: builtins.str = ...,
@@ -1553,7 +1730,7 @@ class AsyncOperator:
root: builtins.str = ...,
table: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `mysql` service.
@@ -1590,7 +1767,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Obs | typing.Literal["obs"],
+ scheme: typing.Literal[opendal.services.Scheme.Obs, "obs"],
/,
*,
access_key_id: builtins.str = ...,
@@ -1599,7 +1776,7 @@ class AsyncOperator:
endpoint: builtins.str = ...,
root: builtins.str = ...,
secret_access_key: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `obs` service.
@@ -1626,7 +1803,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Onedrive | typing.Literal["onedrive"],
+ scheme: typing.Literal[opendal.services.Scheme.Onedrive, "onedrive"],
/,
*,
access_token: builtins.str = ...,
@@ -1635,7 +1812,7 @@ class AsyncOperator:
enable_versioning: builtins.bool = ...,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `onedrive` service.
@@ -1666,7 +1843,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Oss | typing.Literal["oss"],
+ scheme: typing.Literal[opendal.services.Scheme.Oss, "oss"],
/,
*,
access_key_id: builtins.str = ...,
@@ -1689,7 +1866,7 @@ class AsyncOperator:
server_side_encryption: builtins.str = ...,
server_side_encryption_key_id: builtins.str = ...,
sts_endpoint: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `oss` service.
@@ -1698,11 +1875,11 @@ class AsyncOperator:
access_key_id : builtins.str, optional
Access key id for oss.
- this field if it's `is_some` - env value:
- [`ALIBABA_CLOUD_ACCESS_KEY_ID`]
+ `ALIBABA_CLOUD_ACCESS_KEY_ID`
access_key_secret : builtins.str, optional
Access key secret for oss.
- this field if it's `is_some` - env value:
- [`ALIBABA_CLOUD_ACCESS_KEY_SECRET`]
+ `ALIBABA_CLOUD_ACCESS_KEY_SECRET`
addressing_style : builtins.str, optional
Addressing style for oss.
allow_anonymous : builtins.bool, optional
@@ -1720,11 +1897,11 @@ class AsyncOperator:
oidc_provider_arn : builtins.str, optional
`oidc_provider_arn` will be loaded from - this field
if it's `is_some` - env value:
- [`ALIBABA_CLOUD_OIDC_PROVIDER_ARN`]
+ `ALIBABA_CLOUD_OIDC_PROVIDER_ARN`
oidc_token_file : builtins.str, optional
`oidc_token_file` will be loaded from - this field
if it's `is_some` - env value:
- [`ALIBABA_CLOUD_OIDC_TOKEN_FILE`]
+ `ALIBABA_CLOUD_OIDC_TOKEN_FILE`
presign_addressing_style : builtins.str, optional
Pre sign addressing style for oss.
presign_endpoint : builtins.str, optional
@@ -1734,7 +1911,7 @@ class AsyncOperator:
config as source credential to assume role with
`role_arn`.
- this field if it's `is_some` - env value:
- [`ALIBABA_CLOUD_ROLE_ARN`]
+ `ALIBABA_CLOUD_ROLE_ARN`
role_session_name : builtins.str, optional
role_session_name for this backend.
root : builtins.str, optional
@@ -1742,7 +1919,7 @@ class AsyncOperator:
security_token : builtins.str, optional
`security_token` will be loaded from - this field if
it's `is_some` - env value:
- [`ALIBABA_CLOUD_SECURITY_TOKEN`]
+ `ALIBABA_CLOUD_SECURITY_TOKEN`
server_side_encryption : builtins.str, optional
Server side encryption for oss.
server_side_encryption_key_id : builtins.str, optional
@@ -1750,7 +1927,7 @@ class AsyncOperator:
sts_endpoint : builtins.str, optional
`sts_endpoint` will be loaded from - this field if
it's `is_some` - env value:
- [`ALIBABA_CLOUD_STS_ENDPOINT`]
+ `ALIBABA_CLOUD_STS_ENDPOINT`
Returns
-------
@@ -1760,13 +1937,13 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Persy | typing.Literal["persy"],
+ scheme: typing.Literal[opendal.services.Scheme.Persy, "persy"],
/,
*,
datafile: builtins.str = ...,
index: builtins.str = ...,
segment: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `persy` service.
@@ -1788,7 +1965,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Postgresql | typing.Literal["postgresql"],
+ scheme: typing.Literal[opendal.services.Scheme.Postgresql, "postgresql"],
/,
*,
connection_string: builtins.str = ...,
@@ -1796,7 +1973,7 @@ class AsyncOperator:
root: builtins.str = ...,
table: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `postgresql` service.
@@ -1832,13 +2009,13 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Redb | typing.Literal["redb"],
+ scheme: typing.Literal[opendal.services.Scheme.Redb, "redb"],
/,
*,
datadir: builtins.str = ...,
root: builtins.str = ...,
table: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `redb` service.
@@ -1859,7 +2036,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Redis | typing.Literal["redis"],
+ scheme: typing.Literal[opendal.services.Scheme.Redis, "redis"],
/,
*,
cluster_endpoints: builtins.str = ...,
@@ -1870,7 +2047,7 @@ class AsyncOperator:
password: builtins.str = ...,
root: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `redis` service.
@@ -1914,14 +2091,17 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.S3 | typing.Literal["s3"],
+ scheme: typing.Literal[opendal.services.Scheme.S3, "s3"],
/,
*,
access_key_id: builtins.str = ...,
allow_anonymous: builtins.bool = ...,
+ assume_role_duration_seconds: builtins.int = ...,
+ assume_role_session_tags: builtins.dict = ...,
batch_max_operations: builtins.int = ...,
bucket: builtins.str,
checksum_algorithm: builtins.str = ...,
+ default_acl: builtins.str = ...,
default_storage_class: builtins.str = ...,
delete_max_size: builtins.int = ...,
disable_config_load: builtins.bool = ...,
@@ -1946,7 +2126,7 @@ class AsyncOperator:
server_side_encryption_customer_key: builtins.str = ...,
server_side_encryption_customer_key_md5: builtins.str = ...,
session_token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `s3` service.
@@ -1960,6 +2140,10 @@ class AsyncOperator:
allow_anonymous : builtins.bool, optional
Allow anonymous will allow opendal to send request
without signing when credential is not loaded.
+ assume_role_duration_seconds : builtins.int, optional
+ assume_role_duration_seconds for this backend.
+ assume_role_session_tags : builtins.dict, optional
+ assume_role_session_tags for this backend.
batch_max_operations : builtins.int, optional
Set maximum batch operations of this backend.
Some compatible services have a limit on the number
@@ -1975,7 +2159,11 @@ class AsyncOperator:
HTTP headers.
This is necessary when writing to AWS S3 Buckets
with Object Lock enabled for example.
- Available options: - "crc32c"
+ Available options: - "crc32c" - "md5"
+ default_acl : builtins.str, optional
+ Default ACL for new objects.
+ Note that some s3 services like minio do not support
+ this option.
default_storage_class : builtins.str, optional
default storage_class for this backend.
Available values: - `DEEP_ARCHIVE` - `GLACIER` -
@@ -1999,7 +2187,7 @@ class AsyncOperator:
Disable load credential from ec2 metadata.
This option is used to disable the default behavior
of opendal to load credential from ec2 metadata,
- a.k.a, IMDSv2
+ a.k.a., IMDSv2
disable_list_objects_v2 : builtins.bool, optional
OpenDAL uses List Objects V2 by default to list
objects.
@@ -2015,7 +2203,7 @@ class AsyncOperator:
Disable write with if match so that opendal will not
send write request with if match headers.
For example, Ceph RADOS S3 doesn't support write
- with if match.
+ with if matched.
enable_request_payer : builtins.bool, optional
Indicates whether the client agrees to pay for the
requests made to the S3 bucket.
@@ -2120,7 +2308,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Seafile | typing.Literal["seafile"],
+ scheme: typing.Literal[opendal.services.Scheme.Seafile, "seafile"],
/,
*,
endpoint: builtins.str = ...,
@@ -2128,7 +2316,7 @@ class AsyncOperator:
repo_name: builtins.str,
root: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `seafile` service.
@@ -2155,7 +2343,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Sftp | typing.Literal["sftp"],
+ scheme: typing.Literal[opendal.services.Scheme.Sftp, "sftp"],
/,
*,
enable_copy: builtins.bool = ...,
@@ -2164,7 +2352,7 @@ class AsyncOperator:
known_hosts_strategy: builtins.str = ...,
root: builtins.str = ...,
user: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `sftp` service.
@@ -2191,13 +2379,13 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Sled | typing.Literal["sled"],
+ scheme: typing.Literal[opendal.services.Scheme.Sled, "sled"],
/,
*,
datadir: builtins.str = ...,
root: builtins.str = ...,
tree: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `sled` service.
@@ -2218,7 +2406,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Sqlite | typing.Literal["sqlite"],
+ scheme: typing.Literal[opendal.services.Scheme.Sqlite, "sqlite"],
/,
*,
connection_string: builtins.str = ...,
@@ -2226,7 +2414,7 @@ class AsyncOperator:
root: builtins.str = ...,
table: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `sqlite` service.
@@ -2265,14 +2453,16 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Swift | typing.Literal["swift"],
+ scheme: typing.Literal[opendal.services.Scheme.Swift, "swift"],
/,
*,
container: builtins.str = ...,
endpoint: builtins.str = ...,
root: builtins.str = ...,
+ temp_url_hash_algorithm: builtins.str = ...,
+ temp_url_key: builtins.str = ...,
token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `swift` service.
@@ -2284,6 +2474,18 @@ class AsyncOperator:
The endpoint for Swift.
root : builtins.str, optional
The root for Swift.
+ temp_url_hash_algorithm : builtins.str, optional
+ The hash algorithm for TempURL signing.
+ Supported values: `sha1`, `sha256`, `sha512`.
+ Defaults to `sha256`.
+ The cluster must have the chosen algorithm in its
+ `tempurl.allowed_digests` (check `GET /info`).
+ temp_url_key : builtins.str, optional
+ The TempURL key for generating presigned URLs.
+ This corresponds to the
+ `X-Account-Meta-Temp-URL-Key` or
+ `X-Container-Meta-Temp-URL-Key` header value
+ configured on the Swift account or container.
token : builtins.str, optional
The token for Swift.
@@ -2295,14 +2497,90 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Upyun | typing.Literal["upyun"],
+ scheme: typing.Literal[opendal.services.Scheme.Tos, "tos"],
+ /,
+ *,
+ access_key_id: builtins.str = ...,
+ allow_anonymous: builtins.bool = ...,
+ bucket: builtins.str,
+ disable_config_load: builtins.bool = ...,
+ enable_versioning: builtins.bool = ...,
+ endpoint: builtins.str = ...,
+ region: builtins.str = ...,
+ root: builtins.str = ...,
+ secret_access_key: builtins.str = ...,
+ security_token: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `AsyncOperator` for `tos` service.
+
+ Parameters
+ ----------
+ access_key_id : builtins.str, optional
+ access_key_id of this backend.
+ - If access_key_id is set, we will take user's input
+ first.
+ - If not, we will try to load it from environment.
+ allow_anonymous : builtins.bool, optional
+ Allow anonymous will allow opendal to send request
+ without signing when credential is not loaded.
+ bucket : builtins.str
+ bucket name of this backend.
+ required.
+ disable_config_load : builtins.bool, optional
+ Disable config load so that opendal will not load
+ config from environment.
+ For examples: - envs like `TOS_ACCESS_KEY_ID`
+ enable_versioning : builtins.bool, optional
+ Enable bucket versioning for this backend.
+ If set to true, OpenDAL will support versioned
+ operations like list with versions, read with
+ version, etc.
+ endpoint : builtins.str, optional
+ endpoint of this backend.
+ Endpoint must be full uri, e.g.
+ - TOS: `https://tos-cn-beijing.volces.com` - TOS
+ with region: `https://tos-{region}.volces.com` If
+ user inputs endpoint without scheme like
+ "tos-cn-beijing.volces.com", we will prepend
+ "https://" before it.
+ region : builtins.str, optional
+ Region represent the signing region of this
+ endpoint.
+ Required if endpoint is not provided.
+ - If region is set, we will take user's input first.
+ - If not, we will try to load it from environment.
+ - If still not set, default to `cn-beijing`.
+ root : builtins.str, optional
+ root of this backend.
+ All operations will happen under this root.
+ default to `/` if not set.
+ secret_access_key : builtins.str, optional
+ secret_access_key of this backend.
+ - If secret_access_key is set, we will take user's
+ input first.
+ - If not, we will try to load it from environment.
+ security_token : builtins.str, optional
+ security_token of this backend.
+ This token will expire after sometime, it's
+ recommended to set security_token by hand.
+
+ Returns
+ -------
+ AsyncOperator
+ The new `AsyncOperator` for `tos` service
+ """
+ @typing.overload
+ def __new__(
+ cls,
+ scheme: typing.Literal[opendal.services.Scheme.Upyun, "upyun"],
/,
*,
bucket: builtins.str,
operator: builtins.str = ...,
password: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `upyun` service.
@@ -2326,15 +2604,16 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.VercelArtifacts
- | typing.Literal["vercel-artifacts"],
+ scheme: typing.Literal[
+ opendal.services.Scheme.VercelArtifacts, "vercel-artifacts"
+ ],
/,
*,
access_token: builtins.str = ...,
endpoint: builtins.str = ...,
team_id: builtins.str = ...,
team_slug: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `vercel-artifacts` service.
@@ -2343,11 +2622,16 @@ class AsyncOperator:
access_token : builtins.str, optional
The access token for Vercel.
endpoint : builtins.str, optional
- The endpoint for the Vercel artifacts API. Defaults to ``https://api.vercel.com``.
+ The endpoint for the Vercel artifacts API.
+ Defaults to `https://api.vercel.com`.
team_id : builtins.str, optional
The Vercel team ID.
+ When set, the `teamId` query parameter is appended
+ to all API requests.
team_slug : builtins.str, optional
The Vercel team slug.
+ When set, the `slug` query parameter is appended to
+ all API requests.
Returns
-------
@@ -2357,16 +2641,45 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Webdav | typing.Literal["webdav"],
+ scheme: typing.Literal[opendal.services.Scheme.VercelBlob, "vercel-blob"],
+ /,
+ *,
+ root: builtins.str = ...,
+ token: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `AsyncOperator` for `vercel-blob` service.
+
+ Parameters
+ ----------
+ root : builtins.str, optional
+ root of this backend.
+ All operations will happen under this root.
+ token : builtins.str, optional
+ vercel blob token.
+
+ Returns
+ -------
+ AsyncOperator
+ The new `AsyncOperator` for `vercel-blob` service
+ """
+ @typing.overload
+ def __new__(
+ cls,
+ scheme: typing.Literal[opendal.services.Scheme.Webdav, "webdav"],
/,
*,
disable_copy: builtins.bool = ...,
+ disable_create_dir: builtins.bool = ...,
+ enable_user_metadata: builtins.bool = ...,
endpoint: builtins.str = ...,
password: builtins.str = ...,
root: builtins.str = ...,
token: builtins.str = ...,
+ user_metadata_prefix: builtins.str = ...,
+ user_metadata_uri: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `webdav` service.
@@ -2374,6 +2687,28 @@ class AsyncOperator:
----------
disable_copy : builtins.bool, optional
WebDAV Service doesn't support copy.
+ disable_create_dir : builtins.bool, optional
+ Disable automatic parent directory creation before
+ write operations.
+ By default, OpenDAL creates parent directories using
+ MKCOL before writing files.
+ This requires PROPFIND support to check directory
+ existence.
+ Some WebDAV-compatible servers (e.g., bazel-remote)
+ don't support PROPFIND or don't require explicit
+ directory creation.
+ Enable this option to skip the MKCOL calls and write
+ files directly.
+ Default: false
+ enable_user_metadata : builtins.bool, optional
+ Enable user metadata support via WebDAV PROPPATCH.
+ This feature requires the WebDAV server to support
+ RFC4918 PROPPATCH method.
+ Not all WebDAV servers support this (e.g., nginx's
+ basic WebDAV module doesn't).
+ Only enable this if your server supports PROPPATCH
+ (e.g., Apache mod_dav, Nextcloud).
+ Default: false
endpoint : builtins.str, optional
endpoint of this backend
password : builtins.str, optional
@@ -2382,6 +2717,22 @@ class AsyncOperator:
root of this backend
token : builtins.str, optional
token of this backend
+ user_metadata_prefix : builtins.str, optional
+ The XML namespace prefix for user metadata
+ properties.
+ This prefix is used in PROPPATCH/PROPFIND XML
+ requests.
+ Different servers may require different prefixes.
+ Default: "opendal"
+ user_metadata_uri : builtins.str, optional
+ The XML namespace URI for user metadata properties.
+ This URI uniquely identifies the namespace for
+ custom properties.
+ Different servers may require different namespace
+ URIs.
+ For example, Nextcloud might work better with its
+ own namespace.
+ Default: `https://opendal.apache.org/ns`
username : builtins.str, optional
username of this backend
@@ -2393,7 +2744,7 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Webhdfs | typing.Literal["webhdfs"],
+ scheme: typing.Literal[opendal.services.Scheme.Webhdfs, "webhdfs"],
/,
*,
atomic_write_dir: builtins.str = ...,
@@ -2402,7 +2753,7 @@ class AsyncOperator:
endpoint: builtins.str = ...,
root: builtins.str = ...,
user_name: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `webhdfs` service.
@@ -2429,12 +2780,12 @@ class AsyncOperator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.YandexDisk | typing.Literal["yandex-disk"],
+ scheme: typing.Literal[opendal.services.Scheme.YandexDisk, "yandex-disk"],
/,
*,
access_token: builtins.str,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `yandex-disk` service.
@@ -2516,8 +2867,8 @@ class Operator:
size: builtins.int | None = None,
if_match: builtins.str | None = None,
if_none_match: builtins.str | None = None,
- if_modified_since: datetime.datetime = None,
- if_unmodified_since: datetime.datetime = None,
+ if_modified_since: datetime.datetime | None = None,
+ if_unmodified_since: datetime.datetime | None = None,
content_type: builtins.str | None = None,
cache_control: builtins.str | None = None,
content_disposition: builtins.str | None = None,
@@ -2622,8 +2973,8 @@ class Operator:
version: builtins.str | None = None,
if_match: builtins.str | None = None,
if_none_match: builtins.str | None = None,
- if_modified_since: datetime.datetime = None,
- if_unmodified_since: datetime.datetime = None,
+ if_modified_since: datetime.datetime | None = None,
+ if_unmodified_since: datetime.datetime | None = None,
content_type: builtins.str | None = None,
cache_control: builtins.str | None = None,
content_disposition: builtins.str | None = None,
@@ -2710,7 +3061,13 @@ class Operator:
path : str
The path to the directory.
"""
- def delete(self, path: builtins.str | os.PathLike | pathlib.Path) -> None:
+ def delete(
+ self,
+ path: builtins.str | os.PathLike | pathlib.Path,
+ *,
+ version: builtins.str | None = None,
+ recursive: builtins.bool | None = None,
+ ) -> None:
r"""
Delete a file at the given path.
@@ -2722,6 +3079,10 @@ class Operator:
----------
path : str
The path to the file.
+ version : str, optional
+ The version of the file to delete. Only supported on version-aware backends.
+ recursive : bool, optional
+ If True, delete the path recursively. Only supported on backends that support recursive delete.
"""
def exists(self, path: builtins.str | os.PathLike | pathlib.Path) -> builtins.bool:
r"""
@@ -2833,15 +3194,12 @@ class Operator:
"""
@typing.overload
def __new__(
- cls,
- scheme: builtins.str,
- /,
- **kwargs: builtins.str,
- ) -> typing_extensions.Self: ...
+ cls, scheme: builtins.str, /, **kwargs: builtins.str
+ ) -> typing.Self: ...
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.AliyunDrive | typing.Literal["aliyun-drive"],
+ scheme: typing.Literal[opendal.services.Scheme.AliyunDrive, "aliyun-drive"],
/,
*,
access_token: builtins.str = ...,
@@ -2850,7 +3208,7 @@ class Operator:
drive_type: builtins.str,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `aliyun-drive` service.
@@ -2890,12 +3248,12 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Alluxio | typing.Literal["alluxio"],
+ scheme: typing.Literal[opendal.services.Scheme.Alluxio, "alluxio"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `alluxio` service.
@@ -2918,7 +3276,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Azblob | typing.Literal["azblob"],
+ scheme: typing.Literal[opendal.services.Scheme.Azblob, "azblob"],
/,
*,
account_key: builtins.str = ...,
@@ -2931,7 +3289,7 @@ class Operator:
endpoint: builtins.str = ...,
root: builtins.str = ...,
sas_token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `azblob` service.
@@ -2972,7 +3330,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Azdls | typing.Literal["azdls"],
+ scheme: typing.Literal[opendal.services.Scheme.Azdls, "azdls"],
/,
*,
account_key: builtins.str = ...,
@@ -2980,12 +3338,13 @@ class Operator:
authority_host: builtins.str = ...,
client_id: builtins.str = ...,
client_secret: builtins.str = ...,
+ enable_hns: builtins.bool = ...,
endpoint: builtins.str = ...,
filesystem: builtins.str,
root: builtins.str = ...,
sas_token: builtins.str = ...,
tenant_id: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `azdls` service.
@@ -3008,6 +3367,12 @@ class Operator:
client_secret The client secret of the service
principal.
- required for client_credentials authentication
+ enable_hns : builtins.bool, optional
+ Whether hierarchical namespace (HNS) is enabled for
+ the storage account.
+ When enabled, recursive deletion can use pagination
+ to avoid timeouts on large directories.
+ - default value: `false`
endpoint : builtins.str, optional
Endpoint of this backend.
filesystem : builtins.str
@@ -3029,7 +3394,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Azfile | typing.Literal["azfile"],
+ scheme: typing.Literal[opendal.services.Scheme.Azfile, "azfile"],
/,
*,
account_key: builtins.str = ...,
@@ -3038,7 +3403,7 @@ class Operator:
root: builtins.str = ...,
sas_token: builtins.str = ...,
share_name: builtins.str,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `azfile` service.
@@ -3065,7 +3430,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.B2 | typing.Literal["b2"],
+ scheme: typing.Literal[opendal.services.Scheme.B2, "b2"],
/,
*,
application_key: builtins.str = ...,
@@ -3073,7 +3438,7 @@ class Operator:
bucket: builtins.str,
bucket_id: builtins.str,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `b2` service.
@@ -3107,11 +3472,11 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Cacache | typing.Literal["cacache"],
+ scheme: typing.Literal[opendal.services.Scheme.Cacache, "cacache"],
/,
*,
datadir: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `cacache` service.
@@ -3128,7 +3493,45 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Cos | typing.Literal["cos"],
+ scheme: typing.Literal[opendal.services.Scheme.CloudflareKv, "cloudflare-kv"],
+ /,
+ *,
+ account_id: builtins.str = ...,
+ api_token: builtins.str = ...,
+ default_ttl: typing.Any = ...,
+ namespace_id: builtins.str = ...,
+ root: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `Operator` for `cloudflare-kv` service.
+
+ Parameters
+ ----------
+ account_id : builtins.str, optional
+ The account ID used to authenticate with CloudFlare.
+ Used as URI path parameter.
+ api_token : builtins.str, optional
+ The token used to authenticate with CloudFlare.
+ default_ttl : typing.Any, optional
+ The default ttl for write operations..
+ a human readable duration string see
+ https://docs.rs/humantime/latest/humantime/fn.parse_duration.html
+ for more details
+ namespace_id : builtins.str, optional
+ The namespace ID.
+ Used as URI path parameter.
+ root : builtins.str, optional
+ Root within this backend.
+
+ Returns
+ -------
+ Operator
+ The new `Operator` for `cloudflare-kv` service
+ """
+ @typing.overload
+ def __new__(
+ cls,
+ scheme: typing.Literal[opendal.services.Scheme.Cos, "cos"],
/,
*,
bucket: builtins.str = ...,
@@ -3138,7 +3541,7 @@ class Operator:
root: builtins.str = ...,
secret_id: builtins.str = ...,
secret_key: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `cos` service.
@@ -3168,11 +3571,11 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Dashmap | typing.Literal["dashmap"],
+ scheme: typing.Literal[opendal.services.Scheme.Dashmap, "dashmap"],
/,
*,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `dashmap` service.
@@ -3189,7 +3592,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Dropbox | typing.Literal["dropbox"],
+ scheme: typing.Literal[opendal.services.Scheme.Dropbox, "dropbox"],
/,
*,
access_token: builtins.str = ...,
@@ -3197,7 +3600,7 @@ class Operator:
client_secret: builtins.str = ...,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `dropbox` service.
@@ -3222,12 +3625,67 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Fs | typing.Literal["fs"],
+ scheme: typing.Literal[opendal.services.Scheme.Foyer, "foyer"],
+ /,
+ *,
+ disk_capacity: builtins.int = ...,
+ disk_file_size: builtins.int = ...,
+ disk_path: builtins.str = ...,
+ memory: builtins.int = ...,
+ name: builtins.str = ...,
+ recover_mode: builtins.str = ...,
+ root: builtins.str = ...,
+ shards: builtins.int = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `Operator` for `foyer` service.
+
+ Parameters
+ ----------
+ disk_capacity : builtins.int, optional
+ Disk cache total capacity in bytes.
+ Only used when `disk_path` is set.
+ disk_file_size : builtins.int, optional
+ Individual cache file size in bytes.
+ Default is 1 MiB.
+ Only used when `disk_path` is set.
+ disk_path : builtins.str, optional
+ Disk cache directory path.
+ If set, enables hybrid cache with disk storage.
+ Data will be persisted to this directory when memory
+ cache is full.
+ memory : builtins.int, optional
+ Memory capacity in bytes for the cache.
+ name : builtins.str, optional
+ Name for this cache instance.
+ recover_mode : builtins.str, optional
+ Recovery mode when starting the cache.
+ Valid values: "none" (default), "quiet", "strict".
+ - "none": Don't recover from disk - "quiet": Recover
+ and skip errors - "strict": Recover and panic on
+ errors
+ root : builtins.str, optional
+ Root path of this backend.
+ shards : builtins.int, optional
+ Number of shards for concurrent access.
+ Default is 1.
+ Higher values improve concurrency but increase
+ overhead.
+
+ Returns
+ -------
+ Operator
+ The new `Operator` for `foyer` service
+ """
+ @typing.overload
+ def __new__(
+ cls,
+ scheme: typing.Literal[opendal.services.Scheme.Fs, "fs"],
/,
*,
atomic_write_dir: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `fs` service.
@@ -3246,14 +3704,14 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Ftp | typing.Literal["ftp"],
+ scheme: typing.Literal[opendal.services.Scheme.Ftp, "ftp"],
/,
*,
endpoint: builtins.str = ...,
password: builtins.str = ...,
root: builtins.str = ...,
user: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `ftp` service.
@@ -3276,7 +3734,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Gcs | typing.Literal["gcs"],
+ scheme: typing.Literal[opendal.services.Scheme.Gcs, "gcs"],
/,
*,
allow_anonymous: builtins.bool = ...,
@@ -3292,7 +3750,7 @@ class Operator:
scope: builtins.str = ...,
service_account: builtins.str = ...,
token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `gcs` service.
@@ -3340,7 +3798,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Gdrive | typing.Literal["gdrive"],
+ scheme: typing.Literal[opendal.services.Scheme.Gdrive, "gdrive"],
/,
*,
access_token: builtins.str = ...,
@@ -3348,7 +3806,7 @@ class Operator:
client_secret: builtins.str = ...,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `gdrive` service.
@@ -3373,14 +3831,14 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Ghac | typing.Literal["ghac"],
+ scheme: typing.Literal[opendal.services.Scheme.Ghac, "ghac"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
runtime_token: builtins.str = ...,
version: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `ghac` service.
@@ -3403,7 +3861,74 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Gridfs | typing.Literal["gridfs"],
+ scheme: typing.Literal[opendal.services.Scheme.Goosefs, "goosefs"],
+ /,
+ *,
+ auth_type: builtins.str = ...,
+ auth_username: builtins.str = ...,
+ block_size: builtins.int = ...,
+ chunk_size: builtins.int = ...,
+ master_addr: builtins.str = ...,
+ root: builtins.str = ...,
+ write_type: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `Operator` for `goosefs` service.
+
+ Parameters
+ ----------
+ auth_type : builtins.str, optional
+ Authentication type.
+ Supported values: `"nosasl"`, `"simple"`.
+ Default: `"simple"` — PLAIN SASL with usernam
+ e. `"nosasl"` — skip authentication entirely.
+ auth_username : builtins.str, optional
+ Authentication username.
+ Used in SIMPLE mode as the login identity.
+ Default: current OS user (`$USER` / `$USERNAME`).
+ block_size : builtins.int, optional
+ Block size in bytes for new files (default: 64 MiB).
+ chunk_size : builtins.int, optional
+ Chunk size in bytes for streaming RPCs (default: 1
+ MiB).
+ master_addr : builtins.str, optional
+ Master address(es) in `host:port` format.
+ For single master: `"10.0.0.1:9200"` For HA
+ (comma-separated):
+ `"10.0.0.1:9200,10.0.0.2:9200,10.0.0.3:9200"` When
+ multiple addresses are provided, the client uses
+ `PollingMasterInquireClient` to discover the Primary
+ Master automatically.
+ Resolution precedence at `build()` time (highest →
+ lowest), following `goosefs-sdk`
+ `docs/CLIENT_CONFIGURATION.md` §1:
+ 1. This field (when set on the builder / OpenDAL
+ config map)
+ 2. `GOOSEFS_MASTER_ADDR` environment variable
+ 3. `goosefs.master.rpc.addresses` /
+ `goosefs.master.hostname` in
+ `goosefs-site.properties` `build()` fails with
+ `ConfigInvalid` only when **none** of the above
+ supplies a master address.
+ root : builtins.str, optional
+ Root path of this backend.
+ All operations will happen under this root.
+ Default to `/` if not set.
+ write_type : builtins.str, optional
+ Default write type for new files.
+ Supported values: `"must_cache"`, `"cache_through"`,
+ `"through"`, `"async_through"`.
+ Default: `"must_cache"`.
+
+ Returns
+ -------
+ Operator
+ The new `Operator` for `goosefs` service
+ """
+ @typing.overload
+ def __new__(
+ cls,
+ scheme: typing.Literal[opendal.services.Scheme.Gridfs, "gridfs"],
/,
*,
bucket: builtins.str = ...,
@@ -3411,7 +3936,7 @@ class Operator:
connection_string: builtins.str = ...,
database: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `gridfs` service.
@@ -3440,13 +3965,14 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.HdfsNative | typing.Literal["hdfs-native"],
+ scheme: typing.Literal[opendal.services.Scheme.HdfsNative, "hdfs-native"],
/,
*,
enable_append: builtins.bool = ...,
name_node: builtins.str = ...,
+ options: builtins.dict = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `hdfs-native` service.
@@ -3456,6 +3982,8 @@ class Operator:
enable the append capacity
name_node : builtins.str, optional
name_node of this backend
+ options : builtins.dict, optional
+ other options for hdfs client
root : builtins.str, optional
work dir of this backend
@@ -3467,85 +3995,89 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Http | typing.Literal["http"],
+ scheme: typing.Literal[opendal.services.Scheme.Hf, "hf"],
/,
*,
endpoint: builtins.str = ...,
- password: builtins.str = ...,
+ repo_id: builtins.str = ...,
+ repo_type: builtins.str,
+ revision: builtins.str = ...,
root: builtins.str = ...,
token: builtins.str = ...,
- username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
- Create a new `Operator` for `http` service.
+ Create a new `Operator` for `hf` service.
Parameters
----------
endpoint : builtins.str, optional
- endpoint of this backend
- password : builtins.str, optional
- password of this backend
+ Endpoint of the Hugging Face Hub.
+ Default is "https://huggingface.co".
+ repo_id : builtins.str, optional
+ Repo id of this backend.
+ This is required.
+ repo_type : builtins.str
+ Repo type of this backend.
+ Default is model.
+ Default is model
+ revision : builtins.str, optional
+ Revision of this backend.
+ Default is main.
root : builtins.str, optional
- root of this backend
+ Root of this backend.
+ Can be "/path/to/dir".
+ Default is "/".
token : builtins.str, optional
- token of this backend
- username : builtins.str, optional
- username of this backend
+ Token of this backend.
+ This is optional.
Returns
-------
Operator
- The new `Operator` for `http` service
+ The new `Operator` for `hf` service
"""
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Hf | typing.Literal["hf"],
+ scheme: typing.Literal[opendal.services.Scheme.Http, "http"],
/,
*,
- repo_id: builtins.str = ...,
- repo_type: builtins.str = ...,
- revision: builtins.str = ...,
+ endpoint: builtins.str = ...,
+ password: builtins.str = ...,
root: builtins.str = ...,
token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ username: builtins.str = ...,
+ ) -> typing.Self:
r"""
- Create a new `Operator` for `hf` (Hugging Face) service.
+ Create a new `Operator` for `http` service.
Parameters
----------
- repo_id : builtins.str, optional
- Repo id of this backend.
- This is required.
- repo_type : builtins.str, optional
- Repo type of this backend.
- Default is model.
- Available values: - model - dataset
- revision : builtins.str, optional
- Revision of this backend.
- Default is main.
+ endpoint : builtins.str, optional
+ endpoint of this backend
+ password : builtins.str, optional
+ password of this backend
root : builtins.str, optional
- Root of this backend.
- Can be "/path/to/dir".
- Default is "/".
+ root of this backend
token : builtins.str, optional
- Token of this backend.
- This is optional.
+ token of this backend
+ username : builtins.str, optional
+ username of this backend
Returns
-------
Operator
- The new `Operator` for `hf` (Hugging Face) service
+ The new `Operator` for `http` service
"""
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Ipfs | typing.Literal["ipfs"],
+ scheme: typing.Literal[opendal.services.Scheme.Ipfs, "ipfs"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `ipfs` service.
@@ -3564,12 +4096,12 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Ipmfs | typing.Literal["ipmfs"],
+ scheme: typing.Literal[opendal.services.Scheme.Ipmfs, "ipmfs"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `ipmfs` service.
@@ -3588,14 +4120,14 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Koofr | typing.Literal["koofr"],
+ scheme: typing.Literal[opendal.services.Scheme.Koofr, "koofr"],
/,
*,
email: builtins.str,
endpoint: builtins.str,
password: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `koofr` service.
@@ -3620,7 +4152,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Memcached | typing.Literal["memcached"],
+ scheme: typing.Literal[opendal.services.Scheme.Memcached, "memcached"],
/,
*,
connection_pool_max_size: builtins.int = ...,
@@ -3629,7 +4161,7 @@ class Operator:
password: builtins.str = ...,
root: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `memcached` service.
@@ -3662,11 +4194,11 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Memory | typing.Literal["memory"],
+ scheme: typing.Literal[opendal.services.Scheme.Memory, "memory"],
/,
*,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `memory` service.
@@ -3683,14 +4215,14 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.MiniMoka | typing.Literal["mini-moka"],
+ scheme: typing.Literal[opendal.services.Scheme.MiniMoka, "mini-moka"],
/,
*,
max_capacity: builtins.int = ...,
root: builtins.str = ...,
time_to_idle: builtins.str = ...,
time_to_live: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `mini-moka` service.
@@ -3719,7 +4251,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Moka | typing.Literal["moka"],
+ scheme: typing.Literal[opendal.services.Scheme.Moka, "moka"],
/,
*,
max_capacity: builtins.int = ...,
@@ -3727,7 +4259,7 @@ class Operator:
root: builtins.str = ...,
time_to_idle: builtins.str = ...,
time_to_live: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `moka` service.
@@ -3758,7 +4290,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Mongodb | typing.Literal["mongodb"],
+ scheme: typing.Literal[opendal.services.Scheme.Mongodb, "mongodb"],
/,
*,
collection: builtins.str = ...,
@@ -3767,7 +4299,7 @@ class Operator:
key_field: builtins.str = ...,
root: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `mongodb` service.
@@ -3794,7 +4326,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Mysql | typing.Literal["mysql"],
+ scheme: typing.Literal[opendal.services.Scheme.Mysql, "mysql"],
/,
*,
connection_string: builtins.str = ...,
@@ -3802,7 +4334,7 @@ class Operator:
root: builtins.str = ...,
table: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `mysql` service.
@@ -3839,7 +4371,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Obs | typing.Literal["obs"],
+ scheme: typing.Literal[opendal.services.Scheme.Obs, "obs"],
/,
*,
access_key_id: builtins.str = ...,
@@ -3848,7 +4380,7 @@ class Operator:
endpoint: builtins.str = ...,
root: builtins.str = ...,
secret_access_key: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `obs` service.
@@ -3875,7 +4407,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Onedrive | typing.Literal["onedrive"],
+ scheme: typing.Literal[opendal.services.Scheme.Onedrive, "onedrive"],
/,
*,
access_token: builtins.str = ...,
@@ -3884,7 +4416,7 @@ class Operator:
enable_versioning: builtins.bool = ...,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `onedrive` service.
@@ -3915,7 +4447,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Oss | typing.Literal["oss"],
+ scheme: typing.Literal[opendal.services.Scheme.Oss, "oss"],
/,
*,
access_key_id: builtins.str = ...,
@@ -3938,7 +4470,7 @@ class Operator:
server_side_encryption: builtins.str = ...,
server_side_encryption_key_id: builtins.str = ...,
sts_endpoint: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `oss` service.
@@ -3947,11 +4479,11 @@ class Operator:
access_key_id : builtins.str, optional
Access key id for oss.
- this field if it's `is_some` - env value:
- [`ALIBABA_CLOUD_ACCESS_KEY_ID`]
+ `ALIBABA_CLOUD_ACCESS_KEY_ID`
access_key_secret : builtins.str, optional
Access key secret for oss.
- this field if it's `is_some` - env value:
- [`ALIBABA_CLOUD_ACCESS_KEY_SECRET`]
+ `ALIBABA_CLOUD_ACCESS_KEY_SECRET`
addressing_style : builtins.str, optional
Addressing style for oss.
allow_anonymous : builtins.bool, optional
@@ -3969,11 +4501,11 @@ class Operator:
oidc_provider_arn : builtins.str, optional
`oidc_provider_arn` will be loaded from - this field
if it's `is_some` - env value:
- [`ALIBABA_CLOUD_OIDC_PROVIDER_ARN`]
+ `ALIBABA_CLOUD_OIDC_PROVIDER_ARN`
oidc_token_file : builtins.str, optional
`oidc_token_file` will be loaded from - this field
if it's `is_some` - env value:
- [`ALIBABA_CLOUD_OIDC_TOKEN_FILE`]
+ `ALIBABA_CLOUD_OIDC_TOKEN_FILE`
presign_addressing_style : builtins.str, optional
Pre sign addressing style for oss.
presign_endpoint : builtins.str, optional
@@ -3983,7 +4515,7 @@ class Operator:
config as source credential to assume role with
`role_arn`.
- this field if it's `is_some` - env value:
- [`ALIBABA_CLOUD_ROLE_ARN`]
+ `ALIBABA_CLOUD_ROLE_ARN`
role_session_name : builtins.str, optional
role_session_name for this backend.
root : builtins.str, optional
@@ -3991,7 +4523,7 @@ class Operator:
security_token : builtins.str, optional
`security_token` will be loaded from - this field if
it's `is_some` - env value:
- [`ALIBABA_CLOUD_SECURITY_TOKEN`]
+ `ALIBABA_CLOUD_SECURITY_TOKEN`
server_side_encryption : builtins.str, optional
Server side encryption for oss.
server_side_encryption_key_id : builtins.str, optional
@@ -3999,7 +4531,7 @@ class Operator:
sts_endpoint : builtins.str, optional
`sts_endpoint` will be loaded from - this field if
it's `is_some` - env value:
- [`ALIBABA_CLOUD_STS_ENDPOINT`]
+ `ALIBABA_CLOUD_STS_ENDPOINT`
Returns
-------
@@ -4009,13 +4541,13 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Persy | typing.Literal["persy"],
+ scheme: typing.Literal[opendal.services.Scheme.Persy, "persy"],
/,
*,
datafile: builtins.str = ...,
index: builtins.str = ...,
segment: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `persy` service.
@@ -4037,7 +4569,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Postgresql | typing.Literal["postgresql"],
+ scheme: typing.Literal[opendal.services.Scheme.Postgresql, "postgresql"],
/,
*,
connection_string: builtins.str = ...,
@@ -4045,7 +4577,7 @@ class Operator:
root: builtins.str = ...,
table: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `postgresql` service.
@@ -4081,13 +4613,13 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Redb | typing.Literal["redb"],
+ scheme: typing.Literal[opendal.services.Scheme.Redb, "redb"],
/,
*,
datadir: builtins.str = ...,
root: builtins.str = ...,
table: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `redb` service.
@@ -4108,7 +4640,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Redis | typing.Literal["redis"],
+ scheme: typing.Literal[opendal.services.Scheme.Redis, "redis"],
/,
*,
cluster_endpoints: builtins.str = ...,
@@ -4119,7 +4651,7 @@ class Operator:
password: builtins.str = ...,
root: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `redis` service.
@@ -4163,14 +4695,17 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.S3 | typing.Literal["s3"],
+ scheme: typing.Literal[opendal.services.Scheme.S3, "s3"],
/,
*,
access_key_id: builtins.str = ...,
allow_anonymous: builtins.bool = ...,
+ assume_role_duration_seconds: builtins.int = ...,
+ assume_role_session_tags: builtins.dict = ...,
batch_max_operations: builtins.int = ...,
bucket: builtins.str,
checksum_algorithm: builtins.str = ...,
+ default_acl: builtins.str = ...,
default_storage_class: builtins.str = ...,
delete_max_size: builtins.int = ...,
disable_config_load: builtins.bool = ...,
@@ -4195,7 +4730,7 @@ class Operator:
server_side_encryption_customer_key: builtins.str = ...,
server_side_encryption_customer_key_md5: builtins.str = ...,
session_token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `s3` service.
@@ -4209,6 +4744,10 @@ class Operator:
allow_anonymous : builtins.bool, optional
Allow anonymous will allow opendal to send request
without signing when credential is not loaded.
+ assume_role_duration_seconds : builtins.int, optional
+ assume_role_duration_seconds for this backend.
+ assume_role_session_tags : builtins.dict, optional
+ assume_role_session_tags for this backend.
batch_max_operations : builtins.int, optional
Set maximum batch operations of this backend.
Some compatible services have a limit on the number
@@ -4224,7 +4763,11 @@ class Operator:
HTTP headers.
This is necessary when writing to AWS S3 Buckets
with Object Lock enabled for example.
- Available options: - "crc32c"
+ Available options: - "crc32c" - "md5"
+ default_acl : builtins.str, optional
+ Default ACL for new objects.
+ Note that some s3 services like minio do not support
+ this option.
default_storage_class : builtins.str, optional
default storage_class for this backend.
Available values: - `DEEP_ARCHIVE` - `GLACIER` -
@@ -4248,7 +4791,7 @@ class Operator:
Disable load credential from ec2 metadata.
This option is used to disable the default behavior
of opendal to load credential from ec2 metadata,
- a.k.a, IMDSv2
+ a.k.a., IMDSv2
disable_list_objects_v2 : builtins.bool, optional
OpenDAL uses List Objects V2 by default to list
objects.
@@ -4264,7 +4807,7 @@ class Operator:
Disable write with if match so that opendal will not
send write request with if match headers.
For example, Ceph RADOS S3 doesn't support write
- with if match.
+ with if matched.
enable_request_payer : builtins.bool, optional
Indicates whether the client agrees to pay for the
requests made to the S3 bucket.
@@ -4369,7 +4912,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Seafile | typing.Literal["seafile"],
+ scheme: typing.Literal[opendal.services.Scheme.Seafile, "seafile"],
/,
*,
endpoint: builtins.str = ...,
@@ -4377,7 +4920,7 @@ class Operator:
repo_name: builtins.str,
root: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `seafile` service.
@@ -4404,7 +4947,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Sftp | typing.Literal["sftp"],
+ scheme: typing.Literal[opendal.services.Scheme.Sftp, "sftp"],
/,
*,
enable_copy: builtins.bool = ...,
@@ -4413,7 +4956,7 @@ class Operator:
known_hosts_strategy: builtins.str = ...,
root: builtins.str = ...,
user: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `sftp` service.
@@ -4440,13 +4983,13 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Sled | typing.Literal["sled"],
+ scheme: typing.Literal[opendal.services.Scheme.Sled, "sled"],
/,
*,
datadir: builtins.str = ...,
root: builtins.str = ...,
tree: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `sled` service.
@@ -4467,7 +5010,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Sqlite | typing.Literal["sqlite"],
+ scheme: typing.Literal[opendal.services.Scheme.Sqlite, "sqlite"],
/,
*,
connection_string: builtins.str = ...,
@@ -4475,7 +5018,7 @@ class Operator:
root: builtins.str = ...,
table: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `sqlite` service.
@@ -4514,14 +5057,16 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Swift | typing.Literal["swift"],
+ scheme: typing.Literal[opendal.services.Scheme.Swift, "swift"],
/,
*,
container: builtins.str = ...,
endpoint: builtins.str = ...,
root: builtins.str = ...,
+ temp_url_hash_algorithm: builtins.str = ...,
+ temp_url_key: builtins.str = ...,
token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `swift` service.
@@ -4533,6 +5078,18 @@ class Operator:
The endpoint for Swift.
root : builtins.str, optional
The root for Swift.
+ temp_url_hash_algorithm : builtins.str, optional
+ The hash algorithm for TempURL signing.
+ Supported values: `sha1`, `sha256`, `sha512`.
+ Defaults to `sha256`.
+ The cluster must have the chosen algorithm in its
+ `tempurl.allowed_digests` (check `GET /info`).
+ temp_url_key : builtins.str, optional
+ The TempURL key for generating presigned URLs.
+ This corresponds to the
+ `X-Account-Meta-Temp-URL-Key` or
+ `X-Container-Meta-Temp-URL-Key` header value
+ configured on the Swift account or container.
token : builtins.str, optional
The token for Swift.
@@ -4544,14 +5101,90 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Upyun | typing.Literal["upyun"],
+ scheme: typing.Literal[opendal.services.Scheme.Tos, "tos"],
+ /,
+ *,
+ access_key_id: builtins.str = ...,
+ allow_anonymous: builtins.bool = ...,
+ bucket: builtins.str,
+ disable_config_load: builtins.bool = ...,
+ enable_versioning: builtins.bool = ...,
+ endpoint: builtins.str = ...,
+ region: builtins.str = ...,
+ root: builtins.str = ...,
+ secret_access_key: builtins.str = ...,
+ security_token: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `Operator` for `tos` service.
+
+ Parameters
+ ----------
+ access_key_id : builtins.str, optional
+ access_key_id of this backend.
+ - If access_key_id is set, we will take user's input
+ first.
+ - If not, we will try to load it from environment.
+ allow_anonymous : builtins.bool, optional
+ Allow anonymous will allow opendal to send request
+ without signing when credential is not loaded.
+ bucket : builtins.str
+ bucket name of this backend.
+ required.
+ disable_config_load : builtins.bool, optional
+ Disable config load so that opendal will not load
+ config from environment.
+ For examples: - envs like `TOS_ACCESS_KEY_ID`
+ enable_versioning : builtins.bool, optional
+ Enable bucket versioning for this backend.
+ If set to true, OpenDAL will support versioned
+ operations like list with versions, read with
+ version, etc.
+ endpoint : builtins.str, optional
+ endpoint of this backend.
+ Endpoint must be full uri, e.g.
+ - TOS: `https://tos-cn-beijing.volces.com` - TOS
+ with region: `https://tos-{region}.volces.com` If
+ user inputs endpoint without scheme like
+ "tos-cn-beijing.volces.com", we will prepend
+ "https://" before it.
+ region : builtins.str, optional
+ Region represent the signing region of this
+ endpoint.
+ Required if endpoint is not provided.
+ - If region is set, we will take user's input first.
+ - If not, we will try to load it from environment.
+ - If still not set, default to `cn-beijing`.
+ root : builtins.str, optional
+ root of this backend.
+ All operations will happen under this root.
+ default to `/` if not set.
+ secret_access_key : builtins.str, optional
+ secret_access_key of this backend.
+ - If secret_access_key is set, we will take user's
+ input first.
+ - If not, we will try to load it from environment.
+ security_token : builtins.str, optional
+ security_token of this backend.
+ This token will expire after sometime, it's
+ recommended to set security_token by hand.
+
+ Returns
+ -------
+ Operator
+ The new `Operator` for `tos` service
+ """
+ @typing.overload
+ def __new__(
+ cls,
+ scheme: typing.Literal[opendal.services.Scheme.Upyun, "upyun"],
/,
*,
bucket: builtins.str,
operator: builtins.str = ...,
password: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `upyun` service.
@@ -4575,15 +5208,16 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.VercelArtifacts
- | typing.Literal["vercel-artifacts"],
+ scheme: typing.Literal[
+ opendal.services.Scheme.VercelArtifacts, "vercel-artifacts"
+ ],
/,
*,
access_token: builtins.str = ...,
endpoint: builtins.str = ...,
team_id: builtins.str = ...,
team_slug: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `vercel-artifacts` service.
@@ -4592,11 +5226,16 @@ class Operator:
access_token : builtins.str, optional
The access token for Vercel.
endpoint : builtins.str, optional
- The endpoint for the Vercel artifacts API. Defaults to ``https://api.vercel.com``.
+ The endpoint for the Vercel artifacts API.
+ Defaults to `https://api.vercel.com`.
team_id : builtins.str, optional
The Vercel team ID.
+ When set, the `teamId` query parameter is appended
+ to all API requests.
team_slug : builtins.str, optional
The Vercel team slug.
+ When set, the `slug` query parameter is appended to
+ all API requests.
Returns
-------
@@ -4606,16 +5245,45 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Webdav | typing.Literal["webdav"],
+ scheme: typing.Literal[opendal.services.Scheme.VercelBlob, "vercel-blob"],
+ /,
+ *,
+ root: builtins.str = ...,
+ token: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `Operator` for `vercel-blob` service.
+
+ Parameters
+ ----------
+ root : builtins.str, optional
+ root of this backend.
+ All operations will happen under this root.
+ token : builtins.str, optional
+ vercel blob token.
+
+ Returns
+ -------
+ Operator
+ The new `Operator` for `vercel-blob` service
+ """
+ @typing.overload
+ def __new__(
+ cls,
+ scheme: typing.Literal[opendal.services.Scheme.Webdav, "webdav"],
/,
*,
disable_copy: builtins.bool = ...,
+ disable_create_dir: builtins.bool = ...,
+ enable_user_metadata: builtins.bool = ...,
endpoint: builtins.str = ...,
password: builtins.str = ...,
root: builtins.str = ...,
token: builtins.str = ...,
+ user_metadata_prefix: builtins.str = ...,
+ user_metadata_uri: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `webdav` service.
@@ -4623,6 +5291,28 @@ class Operator:
----------
disable_copy : builtins.bool, optional
WebDAV Service doesn't support copy.
+ disable_create_dir : builtins.bool, optional
+ Disable automatic parent directory creation before
+ write operations.
+ By default, OpenDAL creates parent directories using
+ MKCOL before writing files.
+ This requires PROPFIND support to check directory
+ existence.
+ Some WebDAV-compatible servers (e.g., bazel-remote)
+ don't support PROPFIND or don't require explicit
+ directory creation.
+ Enable this option to skip the MKCOL calls and write
+ files directly.
+ Default: false
+ enable_user_metadata : builtins.bool, optional
+ Enable user metadata support via WebDAV PROPPATCH.
+ This feature requires the WebDAV server to support
+ RFC4918 PROPPATCH method.
+ Not all WebDAV servers support this (e.g., nginx's
+ basic WebDAV module doesn't).
+ Only enable this if your server supports PROPPATCH
+ (e.g., Apache mod_dav, Nextcloud).
+ Default: false
endpoint : builtins.str, optional
endpoint of this backend
password : builtins.str, optional
@@ -4631,6 +5321,22 @@ class Operator:
root of this backend
token : builtins.str, optional
token of this backend
+ user_metadata_prefix : builtins.str, optional
+ The XML namespace prefix for user metadata
+ properties.
+ This prefix is used in PROPPATCH/PROPFIND XML
+ requests.
+ Different servers may require different prefixes.
+ Default: "opendal"
+ user_metadata_uri : builtins.str, optional
+ The XML namespace URI for user metadata properties.
+ This URI uniquely identifies the namespace for
+ custom properties.
+ Different servers may require different namespace
+ URIs.
+ For example, Nextcloud might work better with its
+ own namespace.
+ Default: `https://opendal.apache.org/ns`
username : builtins.str, optional
username of this backend
@@ -4642,7 +5348,7 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.Webhdfs | typing.Literal["webhdfs"],
+ scheme: typing.Literal[opendal.services.Scheme.Webhdfs, "webhdfs"],
/,
*,
atomic_write_dir: builtins.str = ...,
@@ -4651,7 +5357,7 @@ class Operator:
endpoint: builtins.str = ...,
root: builtins.str = ...,
user_name: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `webhdfs` service.
@@ -4678,12 +5384,12 @@ class Operator:
@typing.overload
def __new__(
cls,
- scheme: opendal.services.Scheme.YandexDisk | typing.Literal["yandex-disk"],
+ scheme: typing.Literal[opendal.services.Scheme.YandexDisk, "yandex-disk"],
/,
*,
access_token: builtins.str,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `yandex-disk` service.
diff --git a/bindings/python/python/opendal/services.pyi b/bindings/python/python/opendal/services.pyi
index 4f21316e3..db036d519 100644
--- a/bindings/python/python/opendal/services.pyi
+++ b/bindings/python/python/opendal/services.pyi
@@ -41,18 +41,21 @@ class Scheme(enum.Enum):
Azfile = ...
B2 = ...
Cacache = ...
+ CloudflareKv = ...
Cos = ...
Dashmap = ...
Dropbox = ...
+ Foyer = ...
Fs = ...
Ftp = ...
Gcs = ...
Gdrive = ...
Ghac = ...
+ Goosefs = ...
Gridfs = ...
HdfsNative = ...
- Http = ...
Hf = ...
+ Http = ...
Ipfs = ...
Ipmfs = ...
Koofr = ...
@@ -75,8 +78,10 @@ class Scheme(enum.Enum):
Sled = ...
Sqlite = ...
Swift = ...
+ Tos = ...
Upyun = ...
VercelArtifacts = ...
+ VercelBlob = ...
Webdav = ...
Webhdfs = ...
YandexDisk = ...
diff --git a/bindings/python/ruff.toml b/bindings/python/ruff.toml
index 76cc69f06..cfb67cb27 100644
--- a/bindings/python/ruff.toml
+++ b/bindings/python/ruff.toml
@@ -29,6 +29,9 @@ skip-magic-trailing-comma = false
[lint]
extend-select = ["D400"]
+ignore = [
+ "W505", # generate code contains long line
+]
select = [
"ANN", # flake8-annotations
"B", # flake8-bugbear
diff --git a/bindings/python/src/file.rs b/bindings/python/src/file.rs
index da744eb39..7d352988e 100644
--- a/bindings/python/src/file.rs
+++ b/bindings/python/src/file.rs
@@ -355,8 +355,8 @@ impl File {
Ok(())
}
- pub fn __enter__(slf: Py<Self>) -> Py<Self> {
- slf
+ pub fn __enter__(slf: PyRef<'_, Self>) -> Py<Self> {
+ slf.into()
}
#[allow(unused_variables)]
@@ -704,7 +704,7 @@ impl AsyncFile {
})
}
- #[gen_stub(override_return_type(type_repr="typing_extensions.Self", imports=("typing_extensions")))]
+ #[gen_stub(override_return_type(type_repr="typing.Self", imports=("typing")))]
fn __aenter__<'a>(slf: PyRef<'a, Self>, py: Python<'a>) -> PyResult<Bound<'a, PyAny>> {
let slf = slf.into_py_any(py)?;
future_into_py(py, async move { Ok(slf) })
diff --git a/bindings/python/src/operator.rs b/bindings/python/src/operator.rs
index 69eefed17..e21fbbc0a 100644
--- a/bindings/python/src/operator.rs
+++ b/bindings/python/src/operator.rs
@@ -264,9 +264,9 @@ impl Operator {
size: Option<usize>,
if_match: Option<String>,
if_none_match: Option<String>,
- #[gen_stub(override_type(type_repr = "datetime.datetime", imports=("datetime")))]
+ #[gen_stub(override_type(type_repr = "datetime.datetime | None", imports=("datetime")))]
if_modified_since: Option<jiff::Timestamp>,
- #[gen_stub(override_type(type_repr = "datetime.datetime", imports=("datetime")))]
+ #[gen_stub(override_type(type_repr = "datetime.datetime | None", imports=("datetime")))]
if_unmodified_since: Option<jiff::Timestamp>,
content_type: Option<String>,
cache_control: Option<String>,
@@ -424,9 +424,9 @@ impl Operator {
version: Option<String>,
if_match: Option<String>,
if_none_match: Option<String>,
- #[gen_stub(override_type(type_repr = "datetime.datetime", imports=("datetime")))]
+ #[gen_stub(override_type(type_repr = "datetime.datetime | None", imports=("datetime")))]
if_modified_since: Option<jiff::Timestamp>,
- #[gen_stub(override_type(type_repr = "datetime.datetime", imports=("datetime")))]
+ #[gen_stub(override_type(type_repr = "datetime.datetime | None", imports=("datetime")))]
if_unmodified_since: Option<jiff::Timestamp>,
content_type: Option<String>,
cache_control: Option<String>,
@@ -947,9 +947,9 @@ impl AsyncOperator {
size: Option<usize>,
if_match: Option<String>,
if_none_match: Option<String>,
- #[gen_stub(override_type(type_repr = "datetime.datetime", imports=("datetime")))]
+ #[gen_stub(override_type(type_repr = "datetime.datetime | None", imports=("datetime")))]
if_modified_since: Option<jiff::Timestamp>,
- #[gen_stub(override_type(type_repr = "datetime.datetime", imports=("datetime")))]
+ #[gen_stub(override_type(type_repr = "datetime.datetime | None", imports=("datetime")))]
if_unmodified_since: Option<jiff::Timestamp>,
content_type: Option<String>,
cache_control: Option<String>,
@@ -1133,9 +1133,9 @@ impl AsyncOperator {
version: Option<String>,
if_match: Option<String>,
if_none_match: Option<String>,
- #[gen_stub(override_type(type_repr = "datetime.datetime", imports=("datetime")))]
+ #[gen_stub(override_type(type_repr = "datetime.datetime | None", imports=("datetime")))]
if_modified_since: Option<jiff::Timestamp>,
- #[gen_stub(override_type(type_repr = "datetime.datetime", imports=("datetime")))]
+ #[gen_stub(override_type(type_repr = "datetime.datetime | None", imports=("datetime")))]
if_unmodified_since: Option<jiff::Timestamp>,
content_type: Option<String>,
cache_control: Option<String>,
@@ -1530,9 +1530,9 @@ impl AsyncOperator {
version: Option<String>,
if_match: Option<String>,
if_none_match: Option<String>,
- #[gen_stub(override_type(type_repr = "datetime.datetime", imports=("datetime")))]
+ #[gen_stub(override_type(type_repr = "datetime.datetime | None", imports=("datetime")))]
if_modified_since: Option<jiff::Timestamp>,
- #[gen_stub(override_type(type_repr = "datetime.datetime", imports=("datetime")))]
+ #[gen_stub(override_type(type_repr = "datetime.datetime | None", imports=("datetime")))]
if_unmodified_since: Option<jiff::Timestamp>,
content_type: Option<String>,
cache_control: Option<String>,
@@ -1612,9 +1612,9 @@ impl AsyncOperator {
version: Option<String>,
if_match: Option<String>,
if_none_match: Option<String>,
- #[gen_stub(override_type(type_repr = "datetime.datetime", imports=("datetime")))]
+ #[gen_stub(override_type(type_repr = "datetime.datetime | None", imports=("datetime")))]
if_modified_since: Option<jiff::Timestamp>,
- #[gen_stub(override_type(type_repr = "datetime.datetime", imports=("datetime")))]
+ #[gen_stub(override_type(type_repr = "datetime.datetime | None", imports=("datetime")))]
if_unmodified_since: Option<jiff::Timestamp>,
content_type: Option<String>,
cache_control: Option<String>,
diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs
index ca57573fe..8716ffecc 100644
--- a/bindings/python/src/services.rs
+++ b/bindings/python/src/services.rs
@@ -37,14 +37,13 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
class Operator:
@overload
def __new__(cls,
scheme: builtins.str,
/,
**kwargs: builtins.str,
- ) -> typing_extensions.Self: ...
+ ) -> typing.Self: ...
"#
}
}
@@ -76,12 +75,16 @@ pub enum PyScheme {
B2,
#[cfg(feature = "services-cacache")]
Cacache,
+ #[cfg(feature = "services-cloudflare-kv")]
+ CloudflareKv,
#[cfg(feature = "services-cos")]
Cos,
#[cfg(feature = "services-dashmap")]
Dashmap,
#[cfg(feature = "services-dropbox")]
Dropbox,
+ #[cfg(feature = "services-foyer")]
+ Foyer,
#[cfg(feature = "services-fs")]
Fs,
#[cfg(feature = "services-ftp")]
@@ -90,18 +93,18 @@ pub enum PyScheme {
Gcs,
#[cfg(feature = "services-gdrive")]
Gdrive,
- #[cfg(feature = "services-goosefs")]
- Goosefs,
#[cfg(feature = "services-ghac")]
Ghac,
+ #[cfg(feature = "services-goosefs")]
+ Goosefs,
#[cfg(feature = "services-gridfs")]
Gridfs,
#[cfg(feature = "services-hdfs-native")]
HdfsNative,
- #[cfg(feature = "services-http")]
- Http,
#[cfg(feature = "services-hf")]
Hf,
+ #[cfg(feature = "services-http")]
+ Http,
#[cfg(feature = "services-ipfs")]
Ipfs,
#[cfg(feature = "services-ipmfs")]
@@ -146,10 +149,14 @@ pub enum PyScheme {
Sqlite,
#[cfg(feature = "services-swift")]
Swift,
+ #[cfg(feature = "services-tos")]
+ Tos,
#[cfg(feature = "services-upyun")]
Upyun,
#[cfg(feature = "services-vercel-artifacts")]
VercelArtifacts,
+ #[cfg(feature = "services-vercel-blob")]
+ VercelBlob,
#[cfg(feature = "services-webdav")]
Webdav,
#[cfg(feature = "services-webhdfs")]
@@ -177,12 +184,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.AliyunDrive, typing.Literal["aliyun-drive"]],
+ scheme: typing.Literal[opendal.services.Scheme.AliyunDrive, "aliyun-drive"],
/,
*,
access_token: builtins.str = ...,
@@ -191,7 +197,7 @@ submit! {
drive_type: builtins.str,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `aliyun-drive` service.
@@ -236,17 +242,16 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Alluxio, typing.Literal["alluxio"]],
+ scheme: typing.Literal[opendal.services.Scheme.Alluxio, "alluxio"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `alluxio` service.
@@ -274,12 +279,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Azblob, typing.Literal["azblob"]],
+ scheme: typing.Literal[opendal.services.Scheme.Azblob, "azblob"],
/,
*,
account_key: builtins.str = ...,
@@ -292,7 +296,7 @@ submit! {
endpoint: builtins.str = ...,
root: builtins.str = ...,
sas_token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `azblob` service.
@@ -338,12 +342,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Azdls, typing.Literal["azdls"]],
+ scheme: typing.Literal[opendal.services.Scheme.Azdls, "azdls"],
/,
*,
account_key: builtins.str = ...,
@@ -351,12 +354,13 @@ submit! {
authority_host: builtins.str = ...,
client_id: builtins.str = ...,
client_secret: builtins.str = ...,
+ enable_hns: builtins.bool = ...,
endpoint: builtins.str = ...,
filesystem: builtins.str,
root: builtins.str = ...,
sas_token: builtins.str = ...,
tenant_id: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `azdls` service.
@@ -379,6 +383,12 @@ submit! {
client_secret The client secret of the service
principal.
- required for client_credentials authentication
+ enable_hns : builtins.bool, optional
+ Whether hierarchical namespace (HNS) is enabled for
+ the storage account.
+ When enabled, recursive deletion can use pagination
+ to avoid timeouts on large directories.
+ - default value: `false`
endpoint : builtins.str, optional
Endpoint of this backend.
filesystem : builtins.str
@@ -405,12 +415,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Azfile, typing.Literal["azfile"]],
+ scheme: typing.Literal[opendal.services.Scheme.Azfile, "azfile"],
/,
*,
account_key: builtins.str = ...,
@@ -419,7 +428,7 @@ submit! {
root: builtins.str = ...,
sas_token: builtins.str = ...,
share_name: builtins.str,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `azfile` service.
@@ -451,12 +460,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.B2, typing.Literal["b2"]],
+ scheme: typing.Literal[opendal.services.Scheme.B2, "b2"],
/,
*,
application_key: builtins.str = ...,
@@ -464,7 +472,7 @@ submit! {
bucket: builtins.str,
bucket_id: builtins.str,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `b2` service.
@@ -503,16 +511,15 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Cacache, typing.Literal["cacache"]],
+ scheme: typing.Literal[opendal.services.Scheme.Cacache, "cacache"],
/,
*,
datadir: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `cacache` service.
@@ -534,12 +541,58 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Cos, typing.Literal["cos"]],
+ scheme: typing.Literal[opendal.services.Scheme.CloudflareKv, "cloudflare-kv"],
+ /,
+ *,
+ account_id: builtins.str = ...,
+ api_token: builtins.str = ...,
+ default_ttl: typing.Any = ...,
+ namespace_id: builtins.str = ...,
+ root: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `Operator` for `cloudflare-kv` service.
+
+ Parameters
+ ----------
+ account_id : builtins.str, optional
+ The account ID used to authenticate with CloudFlare.
+ Used as URI path parameter.
+ api_token : builtins.str, optional
+ The token used to authenticate with CloudFlare.
+ default_ttl : typing.Any, optional
+ The default ttl for write operations..
+ a human readable duration string see
+ https://docs.rs/humantime/latest/humantime/fn.parse_duration.html
+ for more details
+ namespace_id : builtins.str, optional
+ The namespace ID.
+ Used as URI path parameter.
+ root : builtins.str, optional
+ Root within this backend.
+ Returns
+ -------
+ Operator
+ The new `Operator` for `cloudflare-kv` service
+ """
+ "#
+ }
+}
+
+submit! {
+ gen_methods_from_python! {
+ r#"
+ import builtins
+ import typing
+ import opendal.services
+ class Operator:
+ @overload
+ def __new__(cls,
+ scheme: typing.Literal[opendal.services.Scheme.Cos, "cos"],
/,
*,
bucket: builtins.str = ...,
@@ -549,7 +602,7 @@ submit! {
root: builtins.str = ...,
secret_id: builtins.str = ...,
secret_key: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `cos` service.
@@ -584,16 +637,15 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Dashmap, typing.Literal["dashmap"]],
+ scheme: typing.Literal[opendal.services.Scheme.Dashmap, "dashmap"],
/,
*,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `dashmap` service.
@@ -615,12 +667,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Dropbox, typing.Literal["dropbox"]],
+ scheme: typing.Literal[opendal.services.Scheme.Dropbox, "dropbox"],
/,
*,
access_token: builtins.str = ...,
@@ -628,7 +679,7 @@ submit! {
client_secret: builtins.str = ...,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `dropbox` service.
@@ -658,17 +709,80 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Fs, typing.Literal["fs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Foyer, "foyer"],
+ /,
+ *,
+ disk_capacity: builtins.int = ...,
+ disk_file_size: builtins.int = ...,
+ disk_path: builtins.str = ...,
+ memory: builtins.int = ...,
+ name: builtins.str = ...,
+ recover_mode: builtins.str = ...,
+ root: builtins.str = ...,
+ shards: builtins.int = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `Operator` for `foyer` service.
+
+ Parameters
+ ----------
+ disk_capacity : builtins.int, optional
+ Disk cache total capacity in bytes.
+ Only used when `disk_path` is set.
+ disk_file_size : builtins.int, optional
+ Individual cache file size in bytes.
+ Default is 1 MiB.
+ Only used when `disk_path` is set.
+ disk_path : builtins.str, optional
+ Disk cache directory path.
+ If set, enables hybrid cache with disk storage.
+ Data will be persisted to this directory when memory
+ cache is full.
+ memory : builtins.int, optional
+ Memory capacity in bytes for the cache.
+ name : builtins.str, optional
+ Name for this cache instance.
+ recover_mode : builtins.str, optional
+ Recovery mode when starting the cache.
+ Valid values: "none" (default), "quiet", "strict".
+ - "none": Don't recover from disk - "quiet": Recover
+ and skip errors - "strict": Recover and panic on
+ errors
+ root : builtins.str, optional
+ Root path of this backend.
+ shards : builtins.int, optional
+ Number of shards for concurrent access.
+ Default is 1.
+ Higher values improve concurrency but increase
+ overhead.
+ Returns
+ -------
+ Operator
+ The new `Operator` for `foyer` service
+ """
+ "#
+ }
+}
+
+submit! {
+ gen_methods_from_python! {
+ r#"
+ import builtins
+ import typing
+ import opendal.services
+ class Operator:
+ @overload
+ def __new__(cls,
+ scheme: typing.Literal[opendal.services.Scheme.Fs, "fs"],
/,
*,
atomic_write_dir: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `fs` service.
@@ -692,19 +806,18 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Ftp, typing.Literal["ftp"]],
+ scheme: typing.Literal[opendal.services.Scheme.Ftp, "ftp"],
/,
*,
endpoint: builtins.str = ...,
password: builtins.str = ...,
root: builtins.str = ...,
user: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `ftp` service.
@@ -732,12 +845,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Gcs, typing.Literal["gcs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Gcs, "gcs"],
/,
*,
allow_anonymous: builtins.bool = ...,
@@ -753,7 +865,7 @@ submit! {
scope: builtins.str = ...,
service_account: builtins.str = ...,
token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `gcs` service.
@@ -806,12 +918,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Gdrive, typing.Literal["gdrive"]],
+ scheme: typing.Literal[opendal.services.Scheme.Gdrive, "gdrive"],
/,
*,
access_token: builtins.str = ...,
@@ -819,7 +930,7 @@ submit! {
client_secret: builtins.str = ...,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `gdrive` service.
@@ -849,19 +960,18 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Ghac, typing.Literal["ghac"]],
+ scheme: typing.Literal[opendal.services.Scheme.Ghac, "ghac"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
runtime_token: builtins.str = ...,
version: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `ghac` service.
@@ -889,12 +999,87 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Gridfs, typing.Literal["gridfs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Goosefs, "goosefs"],
+ /,
+ *,
+ auth_type: builtins.str = ...,
+ auth_username: builtins.str = ...,
+ block_size: builtins.int = ...,
+ chunk_size: builtins.int = ...,
+ master_addr: builtins.str = ...,
+ root: builtins.str = ...,
+ write_type: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `Operator` for `goosefs` service.
+
+ Parameters
+ ----------
+ auth_type : builtins.str, optional
+ Authentication type.
+ Supported values: `"nosasl"`, `"simple"`.
+ Default: `"simple"` — PLAIN SASL with usernam
+ e. `"nosasl"` — skip authentication entirely.
+ auth_username : builtins.str, optional
+ Authentication username.
+ Used in SIMPLE mode as the login identity.
+ Default: current OS user (`$USER` / `$USERNAME`).
+ block_size : builtins.int, optional
+ Block size in bytes for new files (default: 64 MiB).
+ chunk_size : builtins.int, optional
+ Chunk size in bytes for streaming RPCs (default: 1
+ MiB).
+ master_addr : builtins.str, optional
+ Master address(es) in `host:port` format.
+ For single master: `"10.0.0.1:9200"` For HA
+ (comma-separated):
+ `"10.0.0.1:9200,10.0.0.2:9200,10.0.0.3:9200"` When
+ multiple addresses are provided, the client uses
+ `PollingMasterInquireClient` to discover the Primary
+ Master automatically.
+ Resolution precedence at `build()` time (highest →
+ lowest), following `goosefs-sdk`
+ `docs/CLIENT_CONFIGURATION.md` §1:
+ 1. This field (when set on the builder / OpenDAL
+ config map)
+ 2. `GOOSEFS_MASTER_ADDR` environment variable
+ 3. `goosefs.master.rpc.addresses` /
+ `goosefs.master.hostname` in
+ `goosefs-site.properties` `build()` fails with
+ `ConfigInvalid` only when **none** of the above
+ supplies a master address.
+ root : builtins.str, optional
+ Root path of this backend.
+ All operations will happen under this root.
+ Default to `/` if not set.
+ write_type : builtins.str, optional
+ Default write type for new files.
+ Supported values: `"must_cache"`, `"cache_through"`,
+ `"through"`, `"async_through"`.
+ Default: `"must_cache"`.
+ Returns
+ -------
+ Operator
+ The new `Operator` for `goosefs` service
+ """
+ "#
+ }
+}
+
+submit! {
+ gen_methods_from_python! {
+ r#"
+ import builtins
+ import typing
+ import opendal.services
+ class Operator:
+ @overload
+ def __new__(cls,
+ scheme: typing.Literal[opendal.services.Scheme.Gridfs, "gridfs"],
/,
*,
bucket: builtins.str = ...,
@@ -902,7 +1087,7 @@ submit! {
connection_string: builtins.str = ...,
database: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `gridfs` service.
@@ -936,18 +1121,18 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.HdfsNative, typing.Literal["hdfs-native"]],
+ scheme: typing.Literal[opendal.services.Scheme.HdfsNative, "hdfs-native"],
/,
*,
enable_append: builtins.bool = ...,
name_node: builtins.str = ...,
+ options: builtins.dict = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `hdfs-native` service.
@@ -957,6 +1142,8 @@ submit! {
enable the append capacity
name_node : builtins.str, optional
name_node of this backend
+ options : builtins.dict, optional
+ other options for hdfs client
root : builtins.str, optional
work dir of this backend
Returns
@@ -973,39 +1160,49 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Http, typing.Literal["http"]],
+ scheme: typing.Literal[opendal.services.Scheme.Hf, "hf"],
/,
*,
endpoint: builtins.str = ...,
- password: builtins.str = ...,
+ repo_id: builtins.str = ...,
+ repo_type: builtins.str,
+ revision: builtins.str = ...,
root: builtins.str = ...,
token: builtins.str = ...,
- username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
- Create a new `Operator` for `http` service.
+ Create a new `Operator` for `hf` service.
Parameters
----------
endpoint : builtins.str, optional
- endpoint of this backend
- password : builtins.str, optional
- password of this backend
+ Endpoint of the Hugging Face Hub.
+ Default is "https://huggingface.co".
+ repo_id : builtins.str, optional
+ Repo id of this backend.
+ This is required.
+ repo_type : builtins.str
+ Repo type of this backend.
+ Default is model.
+ Default is model
+ revision : builtins.str, optional
+ Revision of this backend.
+ Default is main.
root : builtins.str, optional
- root of this backend
+ Root of this backend.
+ Can be "/path/to/dir".
+ Default is "/".
token : builtins.str, optional
- token of this backend
- username : builtins.str, optional
- username of this backend
+ Token of this backend.
+ This is optional.
Returns
-------
Operator
- The new `Operator` for `http` service
+ The new `Operator` for `hf` service
"""
"#
}
@@ -1016,51 +1213,38 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Hf, typing.Literal["hf"]],
+ scheme: typing.Literal[opendal.services.Scheme.Http, "http"],
/,
*,
endpoint: builtins.str = ...,
- repo_id: builtins.str = ...,
- repo_type: builtins.str = ...,
- revision: builtins.str = ...,
+ password: builtins.str = ...,
root: builtins.str = ...,
token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ username: builtins.str = ...,
+ ) -> typing.Self:
r"""
- Create a new `Operator` for `hf` (Hugging Face) service.
+ Create a new `Operator` for `http` service.
Parameters
----------
endpoint : builtins.str, optional
- Endpoint of the Huggingface Hub.
- Default is "https://huggingface.co".
- repo_id : builtins.str, optional
- Repo id of this backend.
- This is required.
- repo_type : builtins.str, optional
- Repo type of this backend.
- Default is model.
- Available values: - model - dataset - datasets
- (alias for dataset)
- revision : builtins.str, optional
- Revision of this backend.
- Default is main.
+ endpoint of this backend
+ password : builtins.str, optional
+ password of this backend
root : builtins.str, optional
- Root of this backend.
- Can be "/path/to/dir".
- Default is "/".
+ root of this backend
token : builtins.str, optional
- Token of this backend.
- This is optional.
+ token of this backend
+ username : builtins.str, optional
+ username of this backend
Returns
-------
Operator
- The new `Operator` for `hf` (Hugging Face) service
+ The new `Operator` for `http` service
"""
"#
}
@@ -1071,17 +1255,16 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Ipfs, typing.Literal["ipfs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Ipfs, "ipfs"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `ipfs` service.
@@ -1105,17 +1288,16 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Ipmfs, typing.Literal["ipmfs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Ipmfs, "ipmfs"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `ipmfs` service.
@@ -1139,19 +1321,18 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Koofr, typing.Literal["koofr"]],
+ scheme: typing.Literal[opendal.services.Scheme.Koofr, "koofr"],
/,
*,
email: builtins.str,
endpoint: builtins.str,
password: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `koofr` service.
@@ -1181,12 +1362,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Memcached, typing.Literal["memcached"]],
+ scheme: typing.Literal[opendal.services.Scheme.Memcached, "memcached"],
/,
*,
connection_pool_max_size: builtins.int = ...,
@@ -1195,7 +1375,7 @@ submit! {
password: builtins.str = ...,
root: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `memcached` service.
@@ -1233,16 +1413,15 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Memory, typing.Literal["memory"]],
+ scheme: typing.Literal[opendal.services.Scheme.Memory, "memory"],
/,
*,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `memory` service.
@@ -1264,19 +1443,18 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.MiniMoka, typing.Literal["mini-moka"]],
+ scheme: typing.Literal[opendal.services.Scheme.MiniMoka, "mini-moka"],
/,
*,
max_capacity: builtins.int = ...,
root: builtins.str = ...,
time_to_idle: builtins.str = ...,
time_to_live: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `mini-moka` service.
@@ -1310,12 +1488,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Moka, typing.Literal["moka"]],
+ scheme: typing.Literal[opendal.services.Scheme.Moka, "moka"],
/,
*,
max_capacity: builtins.int = ...,
@@ -1323,7 +1500,7 @@ submit! {
root: builtins.str = ...,
time_to_idle: builtins.str = ...,
time_to_live: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `moka` service.
@@ -1359,12 +1536,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Mongodb, typing.Literal["mongodb"]],
+ scheme: typing.Literal[opendal.services.Scheme.Mongodb, "mongodb"],
/,
*,
collection: builtins.str = ...,
@@ -1373,7 +1549,7 @@ submit! {
key_field: builtins.str = ...,
root: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `mongodb` service.
@@ -1405,12 +1581,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Mysql, typing.Literal["mysql"]],
+ scheme: typing.Literal[opendal.services.Scheme.Mysql, "mysql"],
/,
*,
connection_string: builtins.str = ...,
@@ -1418,7 +1593,7 @@ submit! {
root: builtins.str = ...,
table: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `mysql` service.
@@ -1460,12 +1635,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Obs, typing.Literal["obs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Obs, "obs"],
/,
*,
access_key_id: builtins.str = ...,
@@ -1474,7 +1648,7 @@ submit! {
endpoint: builtins.str = ...,
root: builtins.str = ...,
secret_access_key: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `obs` service.
@@ -1506,12 +1680,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Onedrive, typing.Literal["onedrive"]],
+ scheme: typing.Literal[opendal.services.Scheme.Onedrive, "onedrive"],
/,
*,
access_token: builtins.str = ...,
@@ -1520,7 +1693,7 @@ submit! {
enable_versioning: builtins.bool = ...,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `onedrive` service.
@@ -1556,12 +1729,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Oss, typing.Literal["oss"]],
+ scheme: typing.Literal[opendal.services.Scheme.Oss, "oss"],
/,
*,
access_key_id: builtins.str = ...,
@@ -1584,7 +1756,7 @@ submit! {
server_side_encryption: builtins.str = ...,
server_side_encryption_key_id: builtins.str = ...,
sts_endpoint: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `oss` service.
@@ -1593,11 +1765,11 @@ submit! {
access_key_id : builtins.str, optional
Access key id for oss.
- this field if it's `is_some` - env value:
- [`ALIBABA_CLOUD_ACCESS_KEY_ID`]
+ `ALIBABA_CLOUD_ACCESS_KEY_ID`
access_key_secret : builtins.str, optional
Access key secret for oss.
- this field if it's `is_some` - env value:
- [`ALIBABA_CLOUD_ACCESS_KEY_SECRET`]
+ `ALIBABA_CLOUD_ACCESS_KEY_SECRET`
addressing_style : builtins.str, optional
Addressing style for oss.
allow_anonymous : builtins.bool, optional
@@ -1615,11 +1787,11 @@ submit! {
oidc_provider_arn : builtins.str, optional
`oidc_provider_arn` will be loaded from - this field
if it's `is_some` - env value:
- [`ALIBABA_CLOUD_OIDC_PROVIDER_ARN`]
+ `ALIBABA_CLOUD_OIDC_PROVIDER_ARN`
oidc_token_file : builtins.str, optional
`oidc_token_file` will be loaded from - this field
if it's `is_some` - env value:
- [`ALIBABA_CLOUD_OIDC_TOKEN_FILE`]
+ `ALIBABA_CLOUD_OIDC_TOKEN_FILE`
presign_addressing_style : builtins.str, optional
Pre sign addressing style for oss.
presign_endpoint : builtins.str, optional
@@ -1629,7 +1801,7 @@ submit! {
config as source credential to assume role with
`role_arn`.
- this field if it's `is_some` - env value:
- [`ALIBABA_CLOUD_ROLE_ARN`]
+ `ALIBABA_CLOUD_ROLE_ARN`
role_session_name : builtins.str, optional
role_session_name for this backend.
root : builtins.str, optional
@@ -1637,7 +1809,7 @@ submit! {
security_token : builtins.str, optional
`security_token` will be loaded from - this field if
it's `is_some` - env value:
- [`ALIBABA_CLOUD_SECURITY_TOKEN`]
+ `ALIBABA_CLOUD_SECURITY_TOKEN`
server_side_encryption : builtins.str, optional
Server side encryption for oss.
server_side_encryption_key_id : builtins.str, optional
@@ -1645,7 +1817,7 @@ submit! {
sts_endpoint : builtins.str, optional
`sts_endpoint` will be loaded from - this field if
it's `is_some` - env value:
- [`ALIBABA_CLOUD_STS_ENDPOINT`]
+ `ALIBABA_CLOUD_STS_ENDPOINT`
Returns
-------
Operator
@@ -1660,18 +1832,17 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Persy, typing.Literal["persy"]],
+ scheme: typing.Literal[opendal.services.Scheme.Persy, "persy"],
/,
*,
datafile: builtins.str = ...,
index: builtins.str = ...,
segment: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `persy` service.
@@ -1698,12 +1869,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Postgresql, typing.Literal["postgresql"]],
+ scheme: typing.Literal[opendal.services.Scheme.Postgresql, "postgresql"],
/,
*,
connection_string: builtins.str = ...,
@@ -1711,7 +1881,7 @@ submit! {
root: builtins.str = ...,
table: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `postgresql` service.
@@ -1752,18 +1922,17 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Redb, typing.Literal["redb"]],
+ scheme: typing.Literal[opendal.services.Scheme.Redb, "redb"],
/,
*,
datadir: builtins.str = ...,
root: builtins.str = ...,
table: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `redb` service.
@@ -1789,12 +1958,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Redis, typing.Literal["redis"]],
+ scheme: typing.Literal[opendal.services.Scheme.Redis, "redis"],
/,
*,
cluster_endpoints: builtins.str = ...,
@@ -1805,7 +1973,7 @@ submit! {
password: builtins.str = ...,
root: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `redis` service.
@@ -1854,19 +2022,21 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.S3, typing.Literal["s3"]],
+ scheme: typing.Literal[opendal.services.Scheme.S3, "s3"],
/,
*,
access_key_id: builtins.str = ...,
allow_anonymous: builtins.bool = ...,
+ assume_role_duration_seconds: builtins.int = ...,
+ assume_role_session_tags: builtins.dict = ...,
batch_max_operations: builtins.int = ...,
bucket: builtins.str,
checksum_algorithm: builtins.str = ...,
+ default_acl: builtins.str = ...,
default_storage_class: builtins.str = ...,
delete_max_size: builtins.int = ...,
disable_config_load: builtins.bool = ...,
@@ -1891,7 +2061,7 @@ submit! {
server_side_encryption_customer_key: builtins.str = ...,
server_side_encryption_customer_key_md5: builtins.str = ...,
session_token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `s3` service.
@@ -1905,6 +2075,10 @@ submit! {
allow_anonymous : builtins.bool, optional
Allow anonymous will allow opendal to send request
without signing when credential is not loaded.
+ assume_role_duration_seconds : builtins.int, optional
+ assume_role_duration_seconds for this backend.
+ assume_role_session_tags : builtins.dict, optional
+ assume_role_session_tags for this backend.
batch_max_operations : builtins.int, optional
Set maximum batch operations of this backend.
Some compatible services have a limit on the number
@@ -1921,6 +2095,10 @@ submit! {
This is necessary when writing to AWS S3 Buckets
with Object Lock enabled for example.
Available options: - "crc32c" - "md5"
+ default_acl : builtins.str, optional
+ Default ACL for new objects.
+ Note that some s3 services like minio do not support
+ this option.
default_storage_class : builtins.str, optional
default storage_class for this backend.
Available values: - `DEEP_ARCHIVE` - `GLACIER` -
@@ -2070,12 +2248,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Seafile, typing.Literal["seafile"]],
+ scheme: typing.Literal[opendal.services.Scheme.Seafile, "seafile"],
/,
*,
endpoint: builtins.str = ...,
@@ -2083,7 +2260,7 @@ submit! {
repo_name: builtins.str,
root: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `seafile` service.
@@ -2115,12 +2292,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Sftp, typing.Literal["sftp"]],
+ scheme: typing.Literal[opendal.services.Scheme.Sftp, "sftp"],
/,
*,
enable_copy: builtins.bool = ...,
@@ -2129,7 +2305,7 @@ submit! {
known_hosts_strategy: builtins.str = ...,
root: builtins.str = ...,
user: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `sftp` service.
@@ -2161,18 +2337,17 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Sled, typing.Literal["sled"]],
+ scheme: typing.Literal[opendal.services.Scheme.Sled, "sled"],
/,
*,
datadir: builtins.str = ...,
root: builtins.str = ...,
tree: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `sled` service.
@@ -2198,12 +2373,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Sqlite, typing.Literal["sqlite"]],
+ scheme: typing.Literal[opendal.services.Scheme.Sqlite, "sqlite"],
/,
*,
connection_string: builtins.str = ...,
@@ -2211,7 +2385,7 @@ submit! {
root: builtins.str = ...,
table: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `sqlite` service.
@@ -2255,19 +2429,20 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Swift, typing.Literal["swift"]],
+ scheme: typing.Literal[opendal.services.Scheme.Swift, "swift"],
/,
*,
container: builtins.str = ...,
endpoint: builtins.str = ...,
root: builtins.str = ...,
+ temp_url_hash_algorithm: builtins.str = ...,
+ temp_url_key: builtins.str = ...,
token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `swift` service.
@@ -2279,6 +2454,18 @@ submit! {
The endpoint for Swift.
root : builtins.str, optional
The root for Swift.
+ temp_url_hash_algorithm : builtins.str, optional
+ The hash algorithm for TempURL signing.
+ Supported values: `sha1`, `sha256`, `sha512`.
+ Defaults to `sha256`.
+ The cluster must have the chosen algorithm in its
+ `tempurl.allowed_digests` (check `GET /info`).
+ temp_url_key : builtins.str, optional
+ The TempURL key for generating presigned URLs.
+ This corresponds to the
+ `X-Account-Meta-Temp-URL-Key` or
+ `X-Container-Meta-Temp-URL-Key` header value
+ configured on the Swift account or container.
token : builtins.str, optional
The token for Swift.
Returns
@@ -2295,19 +2482,103 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Upyun, typing.Literal["upyun"]],
+ scheme: typing.Literal[opendal.services.Scheme.Tos, "tos"],
+ /,
+ *,
+ access_key_id: builtins.str = ...,
+ allow_anonymous: builtins.bool = ...,
+ bucket: builtins.str,
+ disable_config_load: builtins.bool = ...,
+ enable_versioning: builtins.bool = ...,
+ endpoint: builtins.str = ...,
+ region: builtins.str = ...,
+ root: builtins.str = ...,
+ secret_access_key: builtins.str = ...,
+ security_token: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `Operator` for `tos` service.
+
+ Parameters
+ ----------
+ access_key_id : builtins.str, optional
+ access_key_id of this backend.
+ - If access_key_id is set, we will take user's input
+ first.
+ - If not, we will try to load it from environment.
+ allow_anonymous : builtins.bool, optional
+ Allow anonymous will allow opendal to send request
+ without signing when credential is not loaded.
+ bucket : builtins.str
+ bucket name of this backend.
+ required.
+ disable_config_load : builtins.bool, optional
+ Disable config load so that opendal will not load
+ config from environment.
+ For examples: - envs like `TOS_ACCESS_KEY_ID`
+ enable_versioning : builtins.bool, optional
+ Enable bucket versioning for this backend.
+ If set to true, OpenDAL will support versioned
+ operations like list with versions, read with
+ version, etc.
+ endpoint : builtins.str, optional
+ endpoint of this backend.
+ Endpoint must be full uri, e.g.
+ - TOS: `https://tos-cn-beijing.volces.com` - TOS
+ with region: `https://tos-{region}.volces.com` If
+ user inputs endpoint without scheme like
+ "tos-cn-beijing.volces.com", we will prepend
+ "https://" before it.
+ region : builtins.str, optional
+ Region represent the signing region of this
+ endpoint.
+ Required if endpoint is not provided.
+ - If region is set, we will take user's input first.
+ - If not, we will try to load it from environment.
+ - If still not set, default to `cn-beijing`.
+ root : builtins.str, optional
+ root of this backend.
+ All operations will happen under this root.
+ default to `/` if not set.
+ secret_access_key : builtins.str, optional
+ secret_access_key of this backend.
+ - If secret_access_key is set, we will take user's
+ input first.
+ - If not, we will try to load it from environment.
+ security_token : builtins.str, optional
+ security_token of this backend.
+ This token will expire after sometime, it's
+ recommended to set security_token by hand.
+ Returns
+ -------
+ Operator
+ The new `Operator` for `tos` service
+ """
+ "#
+ }
+}
+
+submit! {
+ gen_methods_from_python! {
+ r#"
+ import builtins
+ import typing
+ import opendal.services
+ class Operator:
+ @overload
+ def __new__(cls,
+ scheme: typing.Literal[opendal.services.Scheme.Upyun, "upyun"],
/,
*,
bucket: builtins.str,
operator: builtins.str = ...,
password: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `upyun` service.
@@ -2336,19 +2607,18 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.VercelArtifacts, typing.Literal["vercel-artifacts"]],
+ scheme: typing.Literal[opendal.services.Scheme.VercelArtifacts, "vercel-artifacts"],
/,
*,
access_token: builtins.str = ...,
endpoint: builtins.str = ...,
team_id: builtins.str = ...,
team_slug: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `vercel-artifacts` service.
@@ -2357,11 +2627,16 @@ submit! {
access_token : builtins.str, optional
The access token for Vercel.
endpoint : builtins.str, optional
- The endpoint for the Vercel artifacts API. Defaults to ``https://api.vercel.com``.
+ The endpoint for the Vercel artifacts API.
+ Defaults to `https://api.vercel.com`.
team_id : builtins.str, optional
The Vercel team ID.
+ When set, the `teamId` query parameter is appended
+ to all API requests.
team_slug : builtins.str, optional
The Vercel team slug.
+ When set, the `slug` query parameter is appended to
+ all API requests.
Returns
-------
Operator
@@ -2376,21 +2651,58 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Webdav, typing.Literal["webdav"]],
+ scheme: typing.Literal[opendal.services.Scheme.VercelBlob, "vercel-blob"],
+ /,
+ *,
+ root: builtins.str = ...,
+ token: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `Operator` for `vercel-blob` service.
+
+ Parameters
+ ----------
+ root : builtins.str, optional
+ root of this backend.
+ All operations will happen under this root.
+ token : builtins.str, optional
+ vercel blob token.
+ Returns
+ -------
+ Operator
+ The new `Operator` for `vercel-blob` service
+ """
+ "#
+ }
+}
+
+submit! {
+ gen_methods_from_python! {
+ r#"
+ import builtins
+ import typing
+ import opendal.services
+ class Operator:
+ @overload
+ def __new__(cls,
+ scheme: typing.Literal[opendal.services.Scheme.Webdav, "webdav"],
/,
*,
disable_copy: builtins.bool = ...,
+ disable_create_dir: builtins.bool = ...,
+ enable_user_metadata: builtins.bool = ...,
endpoint: builtins.str = ...,
password: builtins.str = ...,
root: builtins.str = ...,
token: builtins.str = ...,
+ user_metadata_prefix: builtins.str = ...,
+ user_metadata_uri: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `webdav` service.
@@ -2398,6 +2710,28 @@ submit! {
----------
disable_copy : builtins.bool, optional
WebDAV Service doesn't support copy.
+ disable_create_dir : builtins.bool, optional
+ Disable automatic parent directory creation before
+ write operations.
+ By default, OpenDAL creates parent directories using
+ MKCOL before writing files.
+ This requires PROPFIND support to check directory
+ existence.
+ Some WebDAV-compatible servers (e.g., bazel-remote)
+ don't support PROPFIND or don't require explicit
+ directory creation.
+ Enable this option to skip the MKCOL calls and write
+ files directly.
+ Default: false
+ enable_user_metadata : builtins.bool, optional
+ Enable user metadata support via WebDAV PROPPATCH.
+ This feature requires the WebDAV server to support
+ RFC4918 PROPPATCH method.
+ Not all WebDAV servers support this (e.g., nginx's
+ basic WebDAV module doesn't).
+ Only enable this if your server supports PROPPATCH
+ (e.g., Apache mod_dav, Nextcloud).
+ Default: false
endpoint : builtins.str, optional
endpoint of this backend
password : builtins.str, optional
@@ -2406,6 +2740,22 @@ submit! {
root of this backend
token : builtins.str, optional
token of this backend
+ user_metadata_prefix : builtins.str, optional
+ The XML namespace prefix for user metadata
+ properties.
+ This prefix is used in PROPPATCH/PROPFIND XML
+ requests.
+ Different servers may require different prefixes.
+ Default: "opendal"
+ user_metadata_uri : builtins.str, optional
+ The XML namespace URI for user metadata properties.
+ This URI uniquely identifies the namespace for
+ custom properties.
+ Different servers may require different namespace
+ URIs.
+ For example, Nextcloud might work better with its
+ own namespace.
+ Default: `https://opendal.apache.org/ns`
username : builtins.str, optional
username of this backend
Returns
@@ -2422,12 +2772,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Webhdfs, typing.Literal["webhdfs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Webhdfs, "webhdfs"],
/,
*,
atomic_write_dir: builtins.str = ...,
@@ -2436,7 +2785,7 @@ submit! {
endpoint: builtins.str = ...,
root: builtins.str = ...,
user_name: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `webhdfs` service.
@@ -2468,17 +2817,16 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.YandexDisk, typing.Literal["yandex-disk"]],
+ scheme: typing.Literal[opendal.services.Scheme.YandexDisk, "yandex-disk"],
/,
*,
access_token: builtins.str,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `yandex-disk` service.
@@ -2503,14 +2851,13 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
class AsyncOperator:
@overload
def __new__(cls,
scheme: builtins.str,
/,
**kwargs: builtins.str,
- ) -> typing_extensions.Self: ...
+ ) -> typing.Self: ...
"#
}
}
@@ -2520,12 +2867,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.AliyunDrive, typing.Literal["aliyun-drive"]],
+ scheme: typing.Literal[opendal.services.Scheme.AliyunDrive, "aliyun-drive"],
/,
*,
access_token: builtins.str = ...,
@@ -2534,7 +2880,7 @@ submit! {
drive_type: builtins.str,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `aliyun-drive` service.
@@ -2579,17 +2925,16 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Alluxio, typing.Literal["alluxio"]],
+ scheme: typing.Literal[opendal.services.Scheme.Alluxio, "alluxio"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `alluxio` service.
@@ -2617,12 +2962,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Azblob, typing.Literal["azblob"]],
+ scheme: typing.Literal[opendal.services.Scheme.Azblob, "azblob"],
/,
*,
account_key: builtins.str = ...,
@@ -2635,7 +2979,7 @@ submit! {
endpoint: builtins.str = ...,
root: builtins.str = ...,
sas_token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `azblob` service.
@@ -2681,12 +3025,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Azdls, typing.Literal["azdls"]],
+ scheme: typing.Literal[opendal.services.Scheme.Azdls, "azdls"],
/,
*,
account_key: builtins.str = ...,
@@ -2694,12 +3037,13 @@ submit! {
authority_host: builtins.str = ...,
client_id: builtins.str = ...,
client_secret: builtins.str = ...,
+ enable_hns: builtins.bool = ...,
endpoint: builtins.str = ...,
filesystem: builtins.str,
root: builtins.str = ...,
sas_token: builtins.str = ...,
tenant_id: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `azdls` service.
@@ -2722,6 +3066,12 @@ submit! {
client_secret The client secret of the service
principal.
- required for client_credentials authentication
+ enable_hns : builtins.bool, optional
+ Whether hierarchical namespace (HNS) is enabled for
+ the storage account.
+ When enabled, recursive deletion can use pagination
+ to avoid timeouts on large directories.
+ - default value: `false`
endpoint : builtins.str, optional
Endpoint of this backend.
filesystem : builtins.str
@@ -2748,12 +3098,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Azfile, typing.Literal["azfile"]],
+ scheme: typing.Literal[opendal.services.Scheme.Azfile, "azfile"],
/,
*,
account_key: builtins.str = ...,
@@ -2762,7 +3111,7 @@ submit! {
root: builtins.str = ...,
sas_token: builtins.str = ...,
share_name: builtins.str,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `azfile` service.
@@ -2794,12 +3143,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.B2, typing.Literal["b2"]],
+ scheme: typing.Literal[opendal.services.Scheme.B2, "b2"],
/,
*,
application_key: builtins.str = ...,
@@ -2807,7 +3155,7 @@ submit! {
bucket: builtins.str,
bucket_id: builtins.str,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `b2` service.
@@ -2846,16 +3194,15 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Cacache, typing.Literal["cacache"]],
+ scheme: typing.Literal[opendal.services.Scheme.Cacache, "cacache"],
/,
*,
datadir: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `cacache` service.
@@ -2877,12 +3224,58 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Cos, typing.Literal["cos"]],
+ scheme: typing.Literal[opendal.services.Scheme.CloudflareKv, "cloudflare-kv"],
+ /,
+ *,
+ account_id: builtins.str = ...,
+ api_token: builtins.str = ...,
+ default_ttl: typing.Any = ...,
+ namespace_id: builtins.str = ...,
+ root: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `AsyncOperator` for `cloudflare-kv` service.
+
+ Parameters
+ ----------
+ account_id : builtins.str, optional
+ The account ID used to authenticate with CloudFlare.
+ Used as URI path parameter.
+ api_token : builtins.str, optional
+ The token used to authenticate with CloudFlare.
+ default_ttl : typing.Any, optional
+ The default ttl for write operations..
+ a human readable duration string see
+ https://docs.rs/humantime/latest/humantime/fn.parse_duration.html
+ for more details
+ namespace_id : builtins.str, optional
+ The namespace ID.
+ Used as URI path parameter.
+ root : builtins.str, optional
+ Root within this backend.
+ Returns
+ -------
+ AsyncOperator
+ The new `AsyncOperator` for `cloudflare-kv` service
+ """
+ "#
+ }
+}
+
+submit! {
+ gen_methods_from_python! {
+ r#"
+ import builtins
+ import typing
+ import opendal.services
+ class AsyncOperator:
+ @overload
+ def __new__(cls,
+ scheme: typing.Literal[opendal.services.Scheme.Cos, "cos"],
/,
*,
bucket: builtins.str = ...,
@@ -2892,7 +3285,7 @@ submit! {
root: builtins.str = ...,
secret_id: builtins.str = ...,
secret_key: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `cos` service.
@@ -2927,16 +3320,15 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Dashmap, typing.Literal["dashmap"]],
+ scheme: typing.Literal[opendal.services.Scheme.Dashmap, "dashmap"],
/,
*,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `dashmap` service.
@@ -2958,12 +3350,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Dropbox, typing.Literal["dropbox"]],
+ scheme: typing.Literal[opendal.services.Scheme.Dropbox, "dropbox"],
/,
*,
access_token: builtins.str = ...,
@@ -2971,7 +3362,7 @@ submit! {
client_secret: builtins.str = ...,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `dropbox` service.
@@ -3001,17 +3392,80 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Fs, typing.Literal["fs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Foyer, "foyer"],
+ /,
+ *,
+ disk_capacity: builtins.int = ...,
+ disk_file_size: builtins.int = ...,
+ disk_path: builtins.str = ...,
+ memory: builtins.int = ...,
+ name: builtins.str = ...,
+ recover_mode: builtins.str = ...,
+ root: builtins.str = ...,
+ shards: builtins.int = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `AsyncOperator` for `foyer` service.
+
+ Parameters
+ ----------
+ disk_capacity : builtins.int, optional
+ Disk cache total capacity in bytes.
+ Only used when `disk_path` is set.
+ disk_file_size : builtins.int, optional
+ Individual cache file size in bytes.
+ Default is 1 MiB.
+ Only used when `disk_path` is set.
+ disk_path : builtins.str, optional
+ Disk cache directory path.
+ If set, enables hybrid cache with disk storage.
+ Data will be persisted to this directory when memory
+ cache is full.
+ memory : builtins.int, optional
+ Memory capacity in bytes for the cache.
+ name : builtins.str, optional
+ Name for this cache instance.
+ recover_mode : builtins.str, optional
+ Recovery mode when starting the cache.
+ Valid values: "none" (default), "quiet", "strict".
+ - "none": Don't recover from disk - "quiet": Recover
+ and skip errors - "strict": Recover and panic on
+ errors
+ root : builtins.str, optional
+ Root path of this backend.
+ shards : builtins.int, optional
+ Number of shards for concurrent access.
+ Default is 1.
+ Higher values improve concurrency but increase
+ overhead.
+ Returns
+ -------
+ AsyncOperator
+ The new `AsyncOperator` for `foyer` service
+ """
+ "#
+ }
+}
+
+submit! {
+ gen_methods_from_python! {
+ r#"
+ import builtins
+ import typing
+ import opendal.services
+ class AsyncOperator:
+ @overload
+ def __new__(cls,
+ scheme: typing.Literal[opendal.services.Scheme.Fs, "fs"],
/,
*,
atomic_write_dir: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `fs` service.
@@ -3035,19 +3489,18 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Ftp, typing.Literal["ftp"]],
+ scheme: typing.Literal[opendal.services.Scheme.Ftp, "ftp"],
/,
*,
endpoint: builtins.str = ...,
password: builtins.str = ...,
root: builtins.str = ...,
user: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `ftp` service.
@@ -3075,12 +3528,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Gcs, typing.Literal["gcs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Gcs, "gcs"],
/,
*,
allow_anonymous: builtins.bool = ...,
@@ -3096,7 +3548,7 @@ submit! {
scope: builtins.str = ...,
service_account: builtins.str = ...,
token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `gcs` service.
@@ -3149,12 +3601,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Gdrive, typing.Literal["gdrive"]],
+ scheme: typing.Literal[opendal.services.Scheme.Gdrive, "gdrive"],
/,
*,
access_token: builtins.str = ...,
@@ -3162,7 +3613,7 @@ submit! {
client_secret: builtins.str = ...,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `gdrive` service.
@@ -3192,19 +3643,18 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Ghac, typing.Literal["ghac"]],
+ scheme: typing.Literal[opendal.services.Scheme.Ghac, "ghac"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
runtime_token: builtins.str = ...,
version: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `ghac` service.
@@ -3232,12 +3682,87 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Gridfs, typing.Literal["gridfs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Goosefs, "goosefs"],
+ /,
+ *,
+ auth_type: builtins.str = ...,
+ auth_username: builtins.str = ...,
+ block_size: builtins.int = ...,
+ chunk_size: builtins.int = ...,
+ master_addr: builtins.str = ...,
+ root: builtins.str = ...,
+ write_type: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `AsyncOperator` for `goosefs` service.
+
+ Parameters
+ ----------
+ auth_type : builtins.str, optional
+ Authentication type.
+ Supported values: `"nosasl"`, `"simple"`.
+ Default: `"simple"` — PLAIN SASL with usernam
+ e. `"nosasl"` — skip authentication entirely.
+ auth_username : builtins.str, optional
+ Authentication username.
+ Used in SIMPLE mode as the login identity.
+ Default: current OS user (`$USER` / `$USERNAME`).
+ block_size : builtins.int, optional
+ Block size in bytes for new files (default: 64 MiB).
+ chunk_size : builtins.int, optional
+ Chunk size in bytes for streaming RPCs (default: 1
+ MiB).
+ master_addr : builtins.str, optional
+ Master address(es) in `host:port` format.
+ For single master: `"10.0.0.1:9200"` For HA
+ (comma-separated):
+ `"10.0.0.1:9200,10.0.0.2:9200,10.0.0.3:9200"` When
+ multiple addresses are provided, the client uses
+ `PollingMasterInquireClient` to discover the Primary
+ Master automatically.
+ Resolution precedence at `build()` time (highest →
+ lowest), following `goosefs-sdk`
+ `docs/CLIENT_CONFIGURATION.md` §1:
+ 1. This field (when set on the builder / OpenDAL
+ config map)
+ 2. `GOOSEFS_MASTER_ADDR` environment variable
+ 3. `goosefs.master.rpc.addresses` /
+ `goosefs.master.hostname` in
+ `goosefs-site.properties` `build()` fails with
+ `ConfigInvalid` only when **none** of the above
+ supplies a master address.
+ root : builtins.str, optional
+ Root path of this backend.
+ All operations will happen under this root.
+ Default to `/` if not set.
+ write_type : builtins.str, optional
+ Default write type for new files.
+ Supported values: `"must_cache"`, `"cache_through"`,
+ `"through"`, `"async_through"`.
+ Default: `"must_cache"`.
+ Returns
+ -------
+ AsyncOperator
+ The new `AsyncOperator` for `goosefs` service
+ """
+ "#
+ }
+}
+
+submit! {
+ gen_methods_from_python! {
+ r#"
+ import builtins
+ import typing
+ import opendal.services
+ class AsyncOperator:
+ @overload
+ def __new__(cls,
+ scheme: typing.Literal[opendal.services.Scheme.Gridfs, "gridfs"],
/,
*,
bucket: builtins.str = ...,
@@ -3245,7 +3770,7 @@ submit! {
connection_string: builtins.str = ...,
database: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `gridfs` service.
@@ -3279,18 +3804,18 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.HdfsNative, typing.Literal["hdfs-native"]],
+ scheme: typing.Literal[opendal.services.Scheme.HdfsNative, "hdfs-native"],
/,
*,
enable_append: builtins.bool = ...,
name_node: builtins.str = ...,
+ options: builtins.dict = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `hdfs-native` service.
@@ -3300,6 +3825,8 @@ submit! {
enable the append capacity
name_node : builtins.str, optional
name_node of this backend
+ options : builtins.dict, optional
+ other options for hdfs client
root : builtins.str, optional
work dir of this backend
Returns
@@ -3316,39 +3843,49 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Http, typing.Literal["http"]],
+ scheme: typing.Literal[opendal.services.Scheme.Hf, "hf"],
/,
*,
endpoint: builtins.str = ...,
- password: builtins.str = ...,
+ repo_id: builtins.str = ...,
+ repo_type: builtins.str,
+ revision: builtins.str = ...,
root: builtins.str = ...,
token: builtins.str = ...,
- username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
- Create a new `AsyncOperator` for `http` service.
+ Create a new `AsyncOperator` for `hf` service.
Parameters
----------
endpoint : builtins.str, optional
- endpoint of this backend
- password : builtins.str, optional
- password of this backend
+ Endpoint of the Hugging Face Hub.
+ Default is "https://huggingface.co".
+ repo_id : builtins.str, optional
+ Repo id of this backend.
+ This is required.
+ repo_type : builtins.str
+ Repo type of this backend.
+ Default is model.
+ Default is model
+ revision : builtins.str, optional
+ Revision of this backend.
+ Default is main.
root : builtins.str, optional
- root of this backend
+ Root of this backend.
+ Can be "/path/to/dir".
+ Default is "/".
token : builtins.str, optional
- token of this backend
- username : builtins.str, optional
- username of this backend
+ Token of this backend.
+ This is optional.
Returns
-------
AsyncOperator
- The new `AsyncOperator` for `http` service
+ The new `AsyncOperator` for `hf` service
"""
"#
}
@@ -3359,51 +3896,38 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Hf, typing.Literal["hf"]],
+ scheme: typing.Literal[opendal.services.Scheme.Http, "http"],
/,
*,
endpoint: builtins.str = ...,
- repo_id: builtins.str = ...,
- repo_type: builtins.str = ...,
- revision: builtins.str = ...,
+ password: builtins.str = ...,
root: builtins.str = ...,
token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ username: builtins.str = ...,
+ ) -> typing.Self:
r"""
- Create a new `AsyncOperator` for `hf` (Hugging Face) service.
+ Create a new `AsyncOperator` for `http` service.
Parameters
----------
endpoint : builtins.str, optional
- Endpoint of the Huggingface Hub.
- Default is "https://huggingface.co".
- repo_id : builtins.str, optional
- Repo id of this backend.
- This is required.
- repo_type : builtins.str, optional
- Repo type of this backend.
- Default is model.
- Available values: - model - dataset - datasets
- (alias for dataset)
- revision : builtins.str, optional
- Revision of this backend.
- Default is main.
+ endpoint of this backend
+ password : builtins.str, optional
+ password of this backend
root : builtins.str, optional
- Root of this backend.
- Can be "/path/to/dir".
- Default is "/".
+ root of this backend
token : builtins.str, optional
- Token of this backend.
- This is optional.
+ token of this backend
+ username : builtins.str, optional
+ username of this backend
Returns
-------
AsyncOperator
- The new `AsyncOperator` for `hf` (Hugging Face) service
+ The new `AsyncOperator` for `http` service
"""
"#
}
@@ -3414,17 +3938,16 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Ipfs, typing.Literal["ipfs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Ipfs, "ipfs"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `ipfs` service.
@@ -3448,17 +3971,16 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Ipmfs, typing.Literal["ipmfs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Ipmfs, "ipmfs"],
/,
*,
endpoint: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `ipmfs` service.
@@ -3482,19 +4004,18 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Koofr, typing.Literal["koofr"]],
+ scheme: typing.Literal[opendal.services.Scheme.Koofr, "koofr"],
/,
*,
email: builtins.str,
endpoint: builtins.str,
password: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `koofr` service.
@@ -3524,12 +4045,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Memcached, typing.Literal["memcached"]],
+ scheme: typing.Literal[opendal.services.Scheme.Memcached, "memcached"],
/,
*,
connection_pool_max_size: builtins.int = ...,
@@ -3538,7 +4058,7 @@ submit! {
password: builtins.str = ...,
root: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `memcached` service.
@@ -3576,16 +4096,15 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Memory, typing.Literal["memory"]],
+ scheme: typing.Literal[opendal.services.Scheme.Memory, "memory"],
/,
*,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `memory` service.
@@ -3607,19 +4126,18 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.MiniMoka, typing.Literal["mini-moka"]],
+ scheme: typing.Literal[opendal.services.Scheme.MiniMoka, "mini-moka"],
/,
*,
max_capacity: builtins.int = ...,
root: builtins.str = ...,
time_to_idle: builtins.str = ...,
time_to_live: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `mini-moka` service.
@@ -3653,12 +4171,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Moka, typing.Literal["moka"]],
+ scheme: typing.Literal[opendal.services.Scheme.Moka, "moka"],
/,
*,
max_capacity: builtins.int = ...,
@@ -3666,7 +4183,7 @@ submit! {
root: builtins.str = ...,
time_to_idle: builtins.str = ...,
time_to_live: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `moka` service.
@@ -3702,12 +4219,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Mongodb, typing.Literal["mongodb"]],
+ scheme: typing.Literal[opendal.services.Scheme.Mongodb, "mongodb"],
/,
*,
collection: builtins.str = ...,
@@ -3716,7 +4232,7 @@ submit! {
key_field: builtins.str = ...,
root: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `mongodb` service.
@@ -3748,12 +4264,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Mysql, typing.Literal["mysql"]],
+ scheme: typing.Literal[opendal.services.Scheme.Mysql, "mysql"],
/,
*,
connection_string: builtins.str = ...,
@@ -3761,7 +4276,7 @@ submit! {
root: builtins.str = ...,
table: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `mysql` service.
@@ -3803,12 +4318,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Obs, typing.Literal["obs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Obs, "obs"],
/,
*,
access_key_id: builtins.str = ...,
@@ -3817,7 +4331,7 @@ submit! {
endpoint: builtins.str = ...,
root: builtins.str = ...,
secret_access_key: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `obs` service.
@@ -3849,12 +4363,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Onedrive, typing.Literal["onedrive"]],
+ scheme: typing.Literal[opendal.services.Scheme.Onedrive, "onedrive"],
/,
*,
access_token: builtins.str = ...,
@@ -3863,7 +4376,7 @@ submit! {
enable_versioning: builtins.bool = ...,
refresh_token: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `onedrive` service.
@@ -3899,12 +4412,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Oss, typing.Literal["oss"]],
+ scheme: typing.Literal[opendal.services.Scheme.Oss, "oss"],
/,
*,
access_key_id: builtins.str = ...,
@@ -3927,7 +4439,7 @@ submit! {
server_side_encryption: builtins.str = ...,
server_side_encryption_key_id: builtins.str = ...,
sts_endpoint: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `oss` service.
@@ -3936,11 +4448,11 @@ submit! {
access_key_id : builtins.str, optional
Access key id for oss.
- this field if it's `is_some` - env value:
- [`ALIBABA_CLOUD_ACCESS_KEY_ID`]
+ `ALIBABA_CLOUD_ACCESS_KEY_ID`
access_key_secret : builtins.str, optional
Access key secret for oss.
- this field if it's `is_some` - env value:
- [`ALIBABA_CLOUD_ACCESS_KEY_SECRET`]
+ `ALIBABA_CLOUD_ACCESS_KEY_SECRET`
addressing_style : builtins.str, optional
Addressing style for oss.
allow_anonymous : builtins.bool, optional
@@ -3958,11 +4470,11 @@ submit! {
oidc_provider_arn : builtins.str, optional
`oidc_provider_arn` will be loaded from - this field
if it's `is_some` - env value:
- [`ALIBABA_CLOUD_OIDC_PROVIDER_ARN`]
+ `ALIBABA_CLOUD_OIDC_PROVIDER_ARN`
oidc_token_file : builtins.str, optional
`oidc_token_file` will be loaded from - this field
if it's `is_some` - env value:
- [`ALIBABA_CLOUD_OIDC_TOKEN_FILE`]
+ `ALIBABA_CLOUD_OIDC_TOKEN_FILE`
presign_addressing_style : builtins.str, optional
Pre sign addressing style for oss.
presign_endpoint : builtins.str, optional
@@ -3972,7 +4484,7 @@ submit! {
config as source credential to assume role with
`role_arn`.
- this field if it's `is_some` - env value:
- [`ALIBABA_CLOUD_ROLE_ARN`]
+ `ALIBABA_CLOUD_ROLE_ARN`
role_session_name : builtins.str, optional
role_session_name for this backend.
root : builtins.str, optional
@@ -3980,7 +4492,7 @@ submit! {
security_token : builtins.str, optional
`security_token` will be loaded from - this field if
it's `is_some` - env value:
- [`ALIBABA_CLOUD_SECURITY_TOKEN`]
+ `ALIBABA_CLOUD_SECURITY_TOKEN`
server_side_encryption : builtins.str, optional
Server side encryption for oss.
server_side_encryption_key_id : builtins.str, optional
@@ -3988,7 +4500,7 @@ submit! {
sts_endpoint : builtins.str, optional
`sts_endpoint` will be loaded from - this field if
it's `is_some` - env value:
- [`ALIBABA_CLOUD_STS_ENDPOINT`]
+ `ALIBABA_CLOUD_STS_ENDPOINT`
Returns
-------
AsyncOperator
@@ -4003,18 +4515,17 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Persy, typing.Literal["persy"]],
+ scheme: typing.Literal[opendal.services.Scheme.Persy, "persy"],
/,
*,
datafile: builtins.str = ...,
index: builtins.str = ...,
segment: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `persy` service.
@@ -4041,12 +4552,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Postgresql, typing.Literal["postgresql"]],
+ scheme: typing.Literal[opendal.services.Scheme.Postgresql, "postgresql"],
/,
*,
connection_string: builtins.str = ...,
@@ -4054,7 +4564,7 @@ submit! {
root: builtins.str = ...,
table: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `postgresql` service.
@@ -4095,18 +4605,17 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Redb, typing.Literal["redb"]],
+ scheme: typing.Literal[opendal.services.Scheme.Redb, "redb"],
/,
*,
datadir: builtins.str = ...,
root: builtins.str = ...,
table: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `redb` service.
@@ -4132,12 +4641,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Redis, typing.Literal["redis"]],
+ scheme: typing.Literal[opendal.services.Scheme.Redis, "redis"],
/,
*,
cluster_endpoints: builtins.str = ...,
@@ -4148,7 +4656,7 @@ submit! {
password: builtins.str = ...,
root: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `redis` service.
@@ -4197,19 +4705,21 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.S3, typing.Literal["s3"]],
+ scheme: typing.Literal[opendal.services.Scheme.S3, "s3"],
/,
*,
access_key_id: builtins.str = ...,
allow_anonymous: builtins.bool = ...,
+ assume_role_duration_seconds: builtins.int = ...,
+ assume_role_session_tags: builtins.dict = ...,
batch_max_operations: builtins.int = ...,
bucket: builtins.str,
checksum_algorithm: builtins.str = ...,
+ default_acl: builtins.str = ...,
default_storage_class: builtins.str = ...,
delete_max_size: builtins.int = ...,
disable_config_load: builtins.bool = ...,
@@ -4234,7 +4744,7 @@ submit! {
server_side_encryption_customer_key: builtins.str = ...,
server_side_encryption_customer_key_md5: builtins.str = ...,
session_token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `s3` service.
@@ -4248,6 +4758,10 @@ submit! {
allow_anonymous : builtins.bool, optional
Allow anonymous will allow opendal to send request
without signing when credential is not loaded.
+ assume_role_duration_seconds : builtins.int, optional
+ assume_role_duration_seconds for this backend.
+ assume_role_session_tags : builtins.dict, optional
+ assume_role_session_tags for this backend.
batch_max_operations : builtins.int, optional
Set maximum batch operations of this backend.
Some compatible services have a limit on the number
@@ -4264,6 +4778,10 @@ submit! {
This is necessary when writing to AWS S3 Buckets
with Object Lock enabled for example.
Available options: - "crc32c" - "md5"
+ default_acl : builtins.str, optional
+ Default ACL for new objects.
+ Note that some s3 services like minio do not support
+ this option.
default_storage_class : builtins.str, optional
default storage_class for this backend.
Available values: - `DEEP_ARCHIVE` - `GLACIER` -
@@ -4413,12 +4931,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Seafile, typing.Literal["seafile"]],
+ scheme: typing.Literal[opendal.services.Scheme.Seafile, "seafile"],
/,
*,
endpoint: builtins.str = ...,
@@ -4426,7 +4943,7 @@ submit! {
repo_name: builtins.str,
root: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `seafile` service.
@@ -4458,12 +4975,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Sftp, typing.Literal["sftp"]],
+ scheme: typing.Literal[opendal.services.Scheme.Sftp, "sftp"],
/,
*,
enable_copy: builtins.bool = ...,
@@ -4472,7 +4988,7 @@ submit! {
known_hosts_strategy: builtins.str = ...,
root: builtins.str = ...,
user: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `sftp` service.
@@ -4504,18 +5020,17 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Sled, typing.Literal["sled"]],
+ scheme: typing.Literal[opendal.services.Scheme.Sled, "sled"],
/,
*,
datadir: builtins.str = ...,
root: builtins.str = ...,
tree: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `sled` service.
@@ -4541,12 +5056,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Sqlite, typing.Literal["sqlite"]],
+ scheme: typing.Literal[opendal.services.Scheme.Sqlite, "sqlite"],
/,
*,
connection_string: builtins.str = ...,
@@ -4554,7 +5068,7 @@ submit! {
root: builtins.str = ...,
table: builtins.str = ...,
value_field: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `sqlite` service.
@@ -4598,19 +5112,20 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Swift, typing.Literal["swift"]],
+ scheme: typing.Literal[opendal.services.Scheme.Swift, "swift"],
/,
*,
container: builtins.str = ...,
endpoint: builtins.str = ...,
root: builtins.str = ...,
+ temp_url_hash_algorithm: builtins.str = ...,
+ temp_url_key: builtins.str = ...,
token: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `swift` service.
@@ -4622,6 +5137,18 @@ submit! {
The endpoint for Swift.
root : builtins.str, optional
The root for Swift.
+ temp_url_hash_algorithm : builtins.str, optional
+ The hash algorithm for TempURL signing.
+ Supported values: `sha1`, `sha256`, `sha512`.
+ Defaults to `sha256`.
+ The cluster must have the chosen algorithm in its
+ `tempurl.allowed_digests` (check `GET /info`).
+ temp_url_key : builtins.str, optional
+ The TempURL key for generating presigned URLs.
+ This corresponds to the
+ `X-Account-Meta-Temp-URL-Key` or
+ `X-Container-Meta-Temp-URL-Key` header value
+ configured on the Swift account or container.
token : builtins.str, optional
The token for Swift.
Returns
@@ -4638,19 +5165,103 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Upyun, typing.Literal["upyun"]],
+ scheme: typing.Literal[opendal.services.Scheme.Tos, "tos"],
+ /,
+ *,
+ access_key_id: builtins.str = ...,
+ allow_anonymous: builtins.bool = ...,
+ bucket: builtins.str,
+ disable_config_load: builtins.bool = ...,
+ enable_versioning: builtins.bool = ...,
+ endpoint: builtins.str = ...,
+ region: builtins.str = ...,
+ root: builtins.str = ...,
+ secret_access_key: builtins.str = ...,
+ security_token: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `AsyncOperator` for `tos` service.
+
+ Parameters
+ ----------
+ access_key_id : builtins.str, optional
+ access_key_id of this backend.
+ - If access_key_id is set, we will take user's input
+ first.
+ - If not, we will try to load it from environment.
+ allow_anonymous : builtins.bool, optional
+ Allow anonymous will allow opendal to send request
+ without signing when credential is not loaded.
+ bucket : builtins.str
+ bucket name of this backend.
+ required.
+ disable_config_load : builtins.bool, optional
+ Disable config load so that opendal will not load
+ config from environment.
+ For examples: - envs like `TOS_ACCESS_KEY_ID`
+ enable_versioning : builtins.bool, optional
+ Enable bucket versioning for this backend.
+ If set to true, OpenDAL will support versioned
+ operations like list with versions, read with
+ version, etc.
+ endpoint : builtins.str, optional
+ endpoint of this backend.
+ Endpoint must be full uri, e.g.
+ - TOS: `https://tos-cn-beijing.volces.com` - TOS
+ with region: `https://tos-{region}.volces.com` If
+ user inputs endpoint without scheme like
+ "tos-cn-beijing.volces.com", we will prepend
+ "https://" before it.
+ region : builtins.str, optional
+ Region represent the signing region of this
+ endpoint.
+ Required if endpoint is not provided.
+ - If region is set, we will take user's input first.
+ - If not, we will try to load it from environment.
+ - If still not set, default to `cn-beijing`.
+ root : builtins.str, optional
+ root of this backend.
+ All operations will happen under this root.
+ default to `/` if not set.
+ secret_access_key : builtins.str, optional
+ secret_access_key of this backend.
+ - If secret_access_key is set, we will take user's
+ input first.
+ - If not, we will try to load it from environment.
+ security_token : builtins.str, optional
+ security_token of this backend.
+ This token will expire after sometime, it's
+ recommended to set security_token by hand.
+ Returns
+ -------
+ AsyncOperator
+ The new `AsyncOperator` for `tos` service
+ """
+ "#
+ }
+}
+
+submit! {
+ gen_methods_from_python! {
+ r#"
+ import builtins
+ import typing
+ import opendal.services
+ class AsyncOperator:
+ @overload
+ def __new__(cls,
+ scheme: typing.Literal[opendal.services.Scheme.Upyun, "upyun"],
/,
*,
bucket: builtins.str,
operator: builtins.str = ...,
password: builtins.str = ...,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `upyun` service.
@@ -4679,19 +5290,18 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.VercelArtifacts, typing.Literal["vercel-artifacts"]],
+ scheme: typing.Literal[opendal.services.Scheme.VercelArtifacts, "vercel-artifacts"],
/,
*,
access_token: builtins.str = ...,
endpoint: builtins.str = ...,
team_id: builtins.str = ...,
team_slug: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `vercel-artifacts` service.
@@ -4700,11 +5310,16 @@ submit! {
access_token : builtins.str, optional
The access token for Vercel.
endpoint : builtins.str, optional
- The endpoint for the Vercel artifacts API. Defaults to ``https://api.vercel.com``.
+ The endpoint for the Vercel artifacts API.
+ Defaults to `https://api.vercel.com`.
team_id : builtins.str, optional
The Vercel team ID.
+ When set, the `teamId` query parameter is appended
+ to all API requests.
team_slug : builtins.str, optional
The Vercel team slug.
+ When set, the `slug` query parameter is appended to
+ all API requests.
Returns
-------
AsyncOperator
@@ -4719,21 +5334,58 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Webdav, typing.Literal["webdav"]],
+ scheme: typing.Literal[opendal.services.Scheme.VercelBlob, "vercel-blob"],
+ /,
+ *,
+ root: builtins.str = ...,
+ token: builtins.str = ...,
+ ) -> typing.Self:
+ r"""
+ Create a new `AsyncOperator` for `vercel-blob` service.
+
+ Parameters
+ ----------
+ root : builtins.str, optional
+ root of this backend.
+ All operations will happen under this root.
+ token : builtins.str, optional
+ vercel blob token.
+ Returns
+ -------
+ AsyncOperator
+ The new `AsyncOperator` for `vercel-blob` service
+ """
+ "#
+ }
+}
+
+submit! {
+ gen_methods_from_python! {
+ r#"
+ import builtins
+ import typing
+ import opendal.services
+ class AsyncOperator:
+ @overload
+ def __new__(cls,
+ scheme: typing.Literal[opendal.services.Scheme.Webdav, "webdav"],
/,
*,
disable_copy: builtins.bool = ...,
+ disable_create_dir: builtins.bool = ...,
+ enable_user_metadata: builtins.bool = ...,
endpoint: builtins.str = ...,
password: builtins.str = ...,
root: builtins.str = ...,
token: builtins.str = ...,
+ user_metadata_prefix: builtins.str = ...,
+ user_metadata_uri: builtins.str = ...,
username: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `webdav` service.
@@ -4741,6 +5393,28 @@ submit! {
----------
disable_copy : builtins.bool, optional
WebDAV Service doesn't support copy.
+ disable_create_dir : builtins.bool, optional
+ Disable automatic parent directory creation before
+ write operations.
+ By default, OpenDAL creates parent directories using
+ MKCOL before writing files.
+ This requires PROPFIND support to check directory
+ existence.
+ Some WebDAV-compatible servers (e.g., bazel-remote)
+ don't support PROPFIND or don't require explicit
+ directory creation.
+ Enable this option to skip the MKCOL calls and write
+ files directly.
+ Default: false
+ enable_user_metadata : builtins.bool, optional
+ Enable user metadata support via WebDAV PROPPATCH.
+ This feature requires the WebDAV server to support
+ RFC4918 PROPPATCH method.
+ Not all WebDAV servers support this (e.g., nginx's
+ basic WebDAV module doesn't).
+ Only enable this if your server supports PROPPATCH
+ (e.g., Apache mod_dav, Nextcloud).
+ Default: false
endpoint : builtins.str, optional
endpoint of this backend
password : builtins.str, optional
@@ -4749,6 +5423,22 @@ submit! {
root of this backend
token : builtins.str, optional
token of this backend
+ user_metadata_prefix : builtins.str, optional
+ The XML namespace prefix for user metadata
+ properties.
+ This prefix is used in PROPPATCH/PROPFIND XML
+ requests.
+ Different servers may require different prefixes.
+ Default: "opendal"
+ user_metadata_uri : builtins.str, optional
+ The XML namespace URI for user metadata properties.
+ This URI uniquely identifies the namespace for
+ custom properties.
+ Different servers may require different namespace
+ URIs.
+ For example, Nextcloud might work better with its
+ own namespace.
+ Default: `https://opendal.apache.org/ns`
username : builtins.str, optional
username of this backend
Returns
@@ -4765,12 +5455,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.Webhdfs, typing.Literal["webhdfs"]],
+ scheme: typing.Literal[opendal.services.Scheme.Webhdfs, "webhdfs"],
/,
*,
atomic_write_dir: builtins.str = ...,
@@ -4779,7 +5468,7 @@ submit! {
endpoint: builtins.str = ...,
root: builtins.str = ...,
user_name: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `webhdfs` service.
@@ -4811,17 +5500,16 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.YandexDisk, typing.Literal["yandex-disk"]],
+ scheme: typing.Literal[opendal.services.Scheme.YandexDisk, "yandex-disk"],
/,
*,
access_token: builtins.str,
root: builtins.str = ...,
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `yandex-disk` service.
@@ -4882,12 +5570,16 @@ impl_enum_to_str!(
B2 => "b2",
#[cfg(feature = "services-cacache")]
Cacache => "cacache",
+ #[cfg(feature = "services-cloudflare-kv")]
+ CloudflareKv => "cloudflare-kv",
#[cfg(feature = "services-cos")]
Cos => "cos",
#[cfg(feature = "services-dashmap")]
Dashmap => "dashmap",
#[cfg(feature = "services-dropbox")]
Dropbox => "dropbox",
+ #[cfg(feature = "services-foyer")]
+ Foyer => "foyer",
#[cfg(feature = "services-fs")]
Fs => "fs",
#[cfg(feature = "services-ftp")]
@@ -4896,18 +5588,18 @@ impl_enum_to_str!(
Gcs => "gcs",
#[cfg(feature = "services-gdrive")]
Gdrive => "gdrive",
- #[cfg(feature = "services-goosefs")]
- Goosefs => "goosefs",
#[cfg(feature = "services-ghac")]
Ghac => "ghac",
+ #[cfg(feature = "services-goosefs")]
+ Goosefs => "goosefs",
#[cfg(feature = "services-gridfs")]
Gridfs => "gridfs",
#[cfg(feature = "services-hdfs-native")]
HdfsNative => "hdfs-native",
- #[cfg(feature = "services-http")]
- Http => "http",
#[cfg(feature = "services-hf")]
Hf => "hf",
+ #[cfg(feature = "services-http")]
+ Http => "http",
#[cfg(feature = "services-ipfs")]
Ipfs => "ipfs",
#[cfg(feature = "services-ipmfs")]
@@ -4952,10 +5644,14 @@ impl_enum_to_str!(
Sqlite => "sqlite",
#[cfg(feature = "services-swift")]
Swift => "swift",
+ #[cfg(feature = "services-tos")]
+ Tos => "tos",
#[cfg(feature = "services-upyun")]
Upyun => "upyun",
#[cfg(feature = "services-vercel-artifacts")]
VercelArtifacts => "vercel-artifacts",
+ #[cfg(feature = "services-vercel-blob")]
+ VercelBlob => "vercel-blob",
#[cfg(feature = "services-webdav")]
Webdav => "webdav",
#[cfg(feature = "services-webhdfs")]
diff --git a/bindings/python/tests/test_async_conditional.py b/bindings/python/tests/test_async_conditional.py
index 089b79331..df2c0e2d5 100644
--- a/bindings/python/tests/test_async_conditional.py
+++ b/bindings/python/tests/test_async_conditional.py
@@ -109,5 +109,4 @@ async def test_async_write_with_if_none_match(service_name, operator, async_oper
if read_back != content:
pytest.skip("backend does not enforce if_none_match on existing files")
-
await async_operator.delete(path)
diff --git a/bindings/python/tests/test_stat_conditional.py b/bindings/python/tests/test_stat_conditional.py
index 1a79604b1..cbb44ce2b 100644
--- a/bindings/python/tests/test_stat_conditional.py
+++ b/bindings/python/tests/test_stat_conditional.py
@@ -32,9 +32,7 @@ def test_stat_accepts_if_match_param(service_name, operator, async_operator):
@pytest.mark.need_capability("write", "stat", "stat_with_if_none_match")
-def test_stat_accepts_if_none_match_param(
- service_name, operator, async_operator
-):
+def test_stat_accepts_if_none_match_param(service_name, operator, async_operator):
path = f"test_stat_if_none_match_{uuid4()}.txt"
operator.write(path, b"test content")
operator.stat(path, if_none_match="etag456")
@@ -84,9 +82,7 @@ async def test_async_stat_accepts_if_none_match_param(
@pytest.mark.need_capability("write", "stat", "stat_with_version")
@pytest.mark.asyncio
-async def test_async_stat_accepts_version_param(
- service_name, operator, async_operator
-):
+async def test_async_stat_accepts_version_param(service_name, operator, async_operator):
path = f"test_async_stat_version_{uuid4()}.txt"
await async_operator.write(path, b"test content")
meta = await async_operator.stat(path)
diff --git a/bindings/python/tests/test_sync_conditional.py b/bindings/python/tests/test_sync_conditional.py
index 54e417f29..14c886466 100644
--- a/bindings/python/tests/test_sync_conditional.py
+++ b/bindings/python/tests/test_sync_conditional.py
@@ -105,5 +105,4 @@ def test_sync_write_with_if_none_match(service_name, operator, async_operator):
if read_back != content:
pytest.skip("backend does not enforce if_none_match on existing files")
-
operator.delete(path)
diff --git a/bindings/python/tests/test_write_conditional.py b/bindings/python/tests/test_write_conditional.py
index c2b36a180..9d1d81768 100644
--- a/bindings/python/tests/test_write_conditional.py
+++ b/bindings/python/tests/test_write_conditional.py
@@ -27,17 +27,13 @@ def test_write_accepts_if_match_param(service_name, operator, async_operator):
@pytest.mark.need_capability("write", "write_with_if_none_match")
-def test_write_accepts_if_none_match_param(
- service_name, operator, async_operator
-):
+def test_write_accepts_if_none_match_param(service_name, operator, async_operator):
path = f"test_write_if_none_match_{uuid4()}.txt"
operator.write(path, b"test content", if_none_match="etag456")
@pytest.mark.need_capability("write", "write_with_if_not_exists")
-def test_write_accepts_if_not_exists_param(
- service_name, operator, async_operator
-):
+def test_write_accepts_if_not_exists_param(service_name, operator, async_operator):
path = f"test_write_if_not_exists_{uuid4()}.txt"
operator.write(path, b"test content", if_not_exists=True)
diff --git a/dev/src/generate/java.rs b/dev/src/generate/java.rs
index 969f6d021..8eb393e83 100644
--- a/dev/src/generate/java.rs
+++ b/dev/src/generate/java.rs
@@ -81,6 +81,7 @@ fn make_field(field: ViaDeserialize<Config>) -> Result<String, minijinja::Error>
ConfigType::Usize | ConfigType::U64 | ConfigType::I64 => "Long",
ConfigType::U32 | ConfigType::U16 => "Integer",
ConfigType::Vec => "List<String>",
+ ConfigType::HashMap => "Map<String, String>",
}
} else {
match field.value {
@@ -90,6 +91,7 @@ fn make_field(field: ViaDeserialize<Config>) -> Result<String, minijinja::Error>
ConfigType::Usize | ConfigType::U64 | ConfigType::I64 => "long",
ConfigType::U32 | ConfigType::U16 => "int",
ConfigType::Vec => "@NonNull List<String>",
+ ConfigType::HashMap => "@NonNull Map<String, String>",
}
};
@@ -134,6 +136,10 @@ fn make_populate_map(field: ViaDeserialize<Config>) -> Result<String, minijinja:
field.name,
case_java_field_name(&field.name)
),
+ ConfigType::HashMap => format!(
+ "map.putAll({});",
+ case_java_field_name(&field.name)
+ ),
};
if field.optional {
diff --git a/dev/src/generate/mod.rs b/dev/src/generate/mod.rs
index 9f5f1298c..b3d52cb8f 100644
--- a/dev/src/generate/mod.rs
+++ b/dev/src/generate/mod.rs
@@ -24,10 +24,19 @@ use anyhow::Result;
pub fn run(language: &str) -> Result<()> {
let workspace_dir = workspace_dir();
- let services_path = workspace_dir
- .join("core/core/src/services")
- .canonicalize()?;
- let services = parser::parse(&services_path.to_string_lossy())?;
+ let mut services = parser::Services::new();
+
+ // Old layout: core/core/src/services/<name>/config.rs (e.g. memory)
+ let old_services_path = workspace_dir.join("core/core/src/services");
+ if old_services_path.exists() {
+ services.extend(parser::parse(&old_services_path.to_string_lossy())?);
+ }
+
+ // New layout: core/services/<name>/src/config.rs (most services)
+ let new_services_path = workspace_dir.join("core/services");
+ if new_services_path.exists() {
+ services.extend(parser::parse(&new_services_path.to_string_lossy())?);
+ }
match language {
"java" => java::generate(workspace_dir, services),
diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs
index 2484eddb4..8d6fb8897 100644
--- a/dev/src/generate/parser.rs
+++ b/dev/src/generate/parser.rs
@@ -18,12 +18,13 @@
use anyhow::{Context, anyhow};
use anyhow::{Result, bail};
use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
use std::fs;
use std::fs::read_dir;
use std::str::FromStr;
use syn::{
Expr, ExprLit, Field, GenericArgument, Item, Lit, LitStr, Meta, PathArguments, Type, TypePath,
+ UseTree,
};
pub type Services = HashMap<String, Service>;
@@ -92,6 +93,9 @@ pub enum ConfigType {
///
/// Please note, all vec in config are `,` separated string.
Vec,
+
+ /// Mapping to rust's `HashMap`
+ HashMap,
}
impl FromStr for ConfigType {
@@ -110,6 +114,7 @@ impl FromStr for ConfigType {
"u16" => ConfigType::U16,
"Vec" => ConfigType::Vec,
+ "HashMap" => ConfigType::HashMap,
v => bail!("unsupported config type {v}"),
})
@@ -147,6 +152,29 @@ pub struct AttrDeprecated {
}
/// List and parse given path to a `Services` struct.
+fn collect_custom_type_names(items: &[Item]) -> HashSet<String> {
+ fn extract_name(tree: &UseTree) -> Option<String> {
+ match tree {
+ UseTree::Name(name) => Some(name.ident.to_string()),
+ UseTree::Path(path) => {
+ let ident = path.tree.as_ref();
+ extract_name(ident)
+ }
+ _ => None,
+ }
+ }
+
+ let mut names = HashSet::new();
+ for item in items {
+ if let Item::Use(item_use) = item {
+ if let Some(name) = extract_name(&item_use.tree) {
+ names.insert(name);
+ }
+ }
+ }
+ names
+}
+
pub fn parse(path: &str) -> Result<Services> {
let mut map = HashMap::default();
@@ -156,6 +184,15 @@ pub fn parse(path: &str) -> Result<Services> {
continue;
}
let path = dir.path().join("config.rs");
+ // Try the old layout (config.rs directly) first, then the new layout (src/config.rs)
+ let path = if path.exists() {
+ path
+ } else {
+ dir.path().join("src/config.rs")
+ };
+ if !path.exists() {
+ continue;
+ }
let content = fs::read_to_string(&path)?;
let parser = ServiceParser {
service: dir.file_name().to_string_lossy().to_string(),
@@ -217,9 +254,11 @@ impl ServiceParser {
})
.ok_or_else(|| anyhow!("there is no Config in {}", &self.path))?;
+ let custom_types = collect_custom_type_names(&ast.items);
+
let mut config = Vec::with_capacity(config_struct.fields.len());
for field in config_struct.fields {
- let field = Self::parse_field(field)?;
+ let field = Self::parse_field(field, &custom_types)?;
config.push(field);
}
@@ -228,7 +267,7 @@ impl ServiceParser {
}
/// TODO: Add comment parse support.
- fn parse_field(field: Field) -> Result<Config> {
+ fn parse_field(field: Field, custom_types: &HashSet<String>) -> Result<Config> {
let name = field
.ident
.clone()
@@ -266,7 +305,11 @@ impl ServiceParser {
segment.ident.to_string()
};
- let typ = type_name.as_str().parse()?;
+ let typ = match type_name.as_str().parse::<ConfigType>() {
+ Ok(t) => t,
+ Err(_) if custom_types.contains(&type_name) => ConfigType::String,
+ Err(_) => bail!("unsupported config type {type_name}"),
+ };
let optional = optional || typ == ConfigType::Bool;
(typ, optional)
@@ -395,7 +438,8 @@ mod tests {
let input = format!("struct Test {{ {input} }}");
let x: ItemStruct = syn::parse_str(&input).unwrap();
let actual =
- ServiceParser::parse_field(x.fields.iter().next().unwrap().clone()).unwrap();
+ ServiceParser::parse_field(x.fields.iter().next().unwrap().clone(), &HashSet::new())
+ .unwrap();
assert_eq!(actual, expected);
}
}
diff --git a/dev/src/generate/python.j2 b/dev/src/generate/python.j2
index fabd4807a..5ef3804a5 100644
--- a/dev/src/generate/python.j2
+++ b/dev/src/generate/python.j2
@@ -37,14 +37,13 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
class Operator:
@overload
def __new__(cls,
scheme: builtins.str,
/,
**kwargs: builtins.str,
- ) -> typing_extensions.Self: ...
+ ) -> typing.Self: ...
"#
}
}
@@ -88,12 +87,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class Operator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.{{ service_to_pascal(srv) }}, typing.Literal["{{ snake_to_kebab_case(srv) }}"]],
+ scheme: typing.Literal[opendal.services.Scheme.{{ service_to_pascal(srv) }}, "{{ snake_to_kebab_case(srv) }}"],
/,
{%- if srvs[srv].config %}
*,
@@ -101,7 +99,7 @@ submit! {
{%- for field in srvs[srv].config %}
{{field.name}}: {{make_python_type(field.value)}}{% if field.optional %} = ...{% endif %},
{%- endfor %}
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `Operator` for `{{ snake_to_kebab_case(srv) }}` service.
{%- if srvs[srv].config %}
@@ -128,14 +126,13 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
class AsyncOperator:
@overload
def __new__(cls,
scheme: builtins.str,
/,
**kwargs: builtins.str,
- ) -> typing_extensions.Self: ...
+ ) -> typing.Self: ...
"#
}
}
@@ -145,12 +142,11 @@ submit! {
r#"
import builtins
import typing
- import typing_extensions
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
- scheme: typing.Union[opendal.services.Scheme.{{ service_to_pascal(srv) }}, typing.Literal["{{ snake_to_kebab_case(srv) }}"]],
+ scheme: typing.Literal[opendal.services.Scheme.{{ service_to_pascal(srv) }}, "{{ snake_to_kebab_case(srv) }}"],
/,
{%- if srvs[srv].config %}
*,
@@ -158,7 +154,7 @@ submit! {
{%- for field in srvs[srv].config %}
{{field.name}}: {{make_python_type(field.value)}}{% if field.optional %} = ...{% endif %},
{%- endfor %}
- ) -> typing_extensions.Self:
+ ) -> typing.Self:
r"""
Create a new `AsyncOperator` for `{{ snake_to_kebab_case(srv) }}` service.
{%- if srvs[srv].config %}
diff --git a/dev/src/generate/python.rs b/dev/src/generate/python.rs
index d4bd7a047..3b3858d38 100644
--- a/dev/src/generate/python.rs
+++ b/dev/src/generate/python.rs
@@ -235,6 +235,7 @@ fn make_python_type(ty: ViaDeserialize<ConfigType>) -> Result<String, minijinja:
| ConfigType::U32
| ConfigType::U16 => "builtins.int",
ConfigType::Vec => "typing.Any",
+ ConfigType::HashMap => "builtins.dict",
ConfigType::String => "builtins.str",
}
.to_string())