You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/11/18 17:20:15 UTC
(incubator-opendal) 01/01: feat: Add benchmark vs aws sdk s3
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch benchmark-s3
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit dd99089341451f8ce344558eb3880289ccc96650
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Sun Nov 19 01:19:53 2023 +0800
feat: Add benchmark vs aws sdk s3
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
Cargo.lock | 644 ++++++++++++++++++++++++++++++++++--
Cargo.toml | 2 +-
core/benches/vs_s3/Cargo.toml | 40 +++
core/benches/vs_s3/README.md | 40 +++
core/benches/vs_s3/src/main.rs | 125 +++++++
core/src/raw/http_util/body.rs | 20 +-
core/src/raw/oio/read/api.rs | 79 ++---
core/src/types/operator/operator.rs | 16 +-
8 files changed, 890 insertions(+), 76 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 662052bd2..f2a7ddfde 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -497,6 +497,383 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5b3469636cdf8543cceab175efca534471f36eee12fb8374aba00eb5e7e7f8a"
+[[package]]
+name = "aws-config"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "84f9625b71b3ee4adbfbca369c6680d156e316ed86d2c7199a2a134563917414"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-runtime",
+ "aws-sdk-sso",
+ "aws-sdk-ssooidc",
+ "aws-sdk-sts",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "fastrand 2.0.1",
+ "hex",
+ "http",
+ "hyper",
+ "ring 0.17.5",
+ "time",
+ "tokio",
+ "tracing",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-credential-types"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5924466398ac76ffd411d297b9d516dcebb0577f7344c0c15fd8e8e04d9c7895"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-http"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb9a3aa335a105a00975c971f1dad403c3175f2a210d98f39345c6af53923912"
+dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "http",
+ "http-body",
+ "pin-project-lite",
+ "tracing",
+]
+
+[[package]]
+name = "aws-runtime"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b75844ecbdf3dc5e0f5ac5fd1088fb1623849990ea9445d2826258ce63be4de5"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-sigv4",
+ "aws-smithy-async",
+ "aws-smithy-eventstream",
+ "aws-smithy-http",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "fastrand 2.0.1",
+ "http",
+ "percent-encoding",
+ "tracing",
+ "uuid",
+]
+
+[[package]]
+name = "aws-sdk-s3"
+version = "0.38.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39a2ccbd49e784c36d0f0596da681275496f679e249059cae3fa92dd3c2c72e1"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-runtime",
+ "aws-sigv4",
+ "aws-smithy-async",
+ "aws-smithy-checksums",
+ "aws-smithy-eventstream",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-smithy-xml",
+ "aws-types",
+ "bytes",
+ "http",
+ "http-body",
+ "once_cell",
+ "percent-encoding",
+ "regex",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "aws-sdk-sso"
+version = "0.38.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c870aa95e1e85f837f74af2cc937b3f8e72e2315a89e524265875843655b4d47"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "http",
+ "regex",
+ "tracing",
+]
+
+[[package]]
+name = "aws-sdk-ssooidc"
+version = "0.38.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "107ee812e46f9120e68d48bf985d2f2a538315bd8be8a3e54db619250cc4c95e"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "http",
+ "regex",
+ "tracing",
+]
+
+[[package]]
+name = "aws-sdk-sts"
+version = "0.38.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4e3958c43a78f6c3822e62009a35802af5cc7c120fbe8e60b98565604569aae"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-query",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-smithy-xml",
+ "aws-types",
+ "http",
+ "regex",
+ "tracing",
+]
+
+[[package]]
+name = "aws-sigv4"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06130e3686db3c5ae2fc44b3516fffe6b4d4eccebe09bd8ccc4067f3c9c183fb"
+dependencies = [
+ "aws-credential-types",
+ "aws-smithy-eventstream",
+ "aws-smithy-http",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "bytes",
+ "crypto-bigint 0.5.4",
+ "form_urlencoded",
+ "hex",
+ "hmac",
+ "http",
+ "once_cell",
+ "p256",
+ "percent-encoding",
+ "regex",
+ "ring 0.17.5",
+ "sha2",
+ "subtle",
+ "time",
+ "tracing",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-smithy-async"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d787b7e07925b450bed90d9d29ac8e57006c9c2ac907151d175ac0e376bfee0e"
+dependencies = [
+ "futures-util",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "aws-smithy-checksums"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d293ef76a982c573f7384e30c28f9a2472f8dd5f4ce5abcceb0e909a15098e8e"
+dependencies = [
+ "aws-smithy-http",
+ "aws-smithy-types",
+ "bytes",
+ "crc32c",
+ "crc32fast",
+ "hex",
+ "http",
+ "http-body",
+ "md-5",
+ "pin-project-lite",
+ "sha1",
+ "sha2",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-eventstream"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1d2c1844aee465dc023924dbe19d730b116eaf3587d7c2a9b4a41f9c4e980ee"
+dependencies = [
+ "aws-smithy-types",
+ "bytes",
+ "crc32fast",
+]
+
+[[package]]
+name = "aws-smithy-http"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96daaad925331c72449423574fdc72b54af780d5a23ace3c0a6ad0ccbf378715"
+dependencies = [
+ "aws-smithy-eventstream",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "bytes",
+ "bytes-utils",
+ "futures-core",
+ "http",
+ "http-body",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "pin-utils",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-json"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ff985bee3fe21046dc501fadc1d04a1161977c55a0cbbccd9b111c18206aa64"
+dependencies = [
+ "aws-smithy-types",
+]
+
+[[package]]
+name = "aws-smithy-query"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb4006503693766d34717efc5f58325062845fce26a683a71b70f23156d72e67"
+dependencies = [
+ "aws-smithy-types",
+ "urlencoding",
+]
+
+[[package]]
+name = "aws-smithy-runtime"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d28af854558601b4202a4273b9720aebe43d73e472143e6056f16e3bd90bc837"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "bytes",
+ "fastrand 2.0.1",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-rustls",
+ "once_cell",
+ "pin-project-lite",
+ "pin-utils",
+ "rustls 0.21.9",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-runtime-api"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1c68e17e754b86da350b43add38294189121a880e9c3fb454f83ff7044f5257"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-types",
+ "bytes",
+ "http",
+ "pin-project-lite",
+ "tokio",
+ "tracing",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-smithy-types"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d97b978d8a351ea5744206ecc643a1d3806628680e9f151b4d6b7a76fec1596f"
+dependencies = [
+ "base64-simd",
+ "bytes",
+ "bytes-utils",
+ "futures-core",
+ "http",
+ "http-body",
+ "itoa",
+ "num-integer",
+ "pin-project-lite",
+ "pin-utils",
+ "ryu",
+ "serde",
+ "time",
+ "tokio",
+ "tokio-util",
+]
+
+[[package]]
+name = "aws-smithy-xml"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97500a0d0884b9576e65076075f81d899cfbb84f7db5ca1dd317f0582204e528"
+dependencies = [
+ "xmlparser",
+]
+
+[[package]]
+name = "aws-types"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61065f0c6070cb0f9aaddfa614605fb1049908481da71ba5b39b2ffca12f57e4"
+dependencies = [
+ "aws-credential-types",
+ "aws-smithy-async",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "http",
+ "rustc_version 0.4.0",
+ "tracing",
+]
+
[[package]]
name = "axum"
version = "0.6.20"
@@ -573,6 +950,12 @@ dependencies = [
"rustc-demangle",
]
+[[package]]
+name = "base16ct"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce"
+
[[package]]
name = "base64"
version = "0.13.1"
@@ -585,6 +968,16 @@ version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2"
+[[package]]
+name = "base64-simd"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195"
+dependencies = [
+ "outref",
+ "vsimd",
+]
+
[[package]]
name = "base64ct"
version = "1.6.0"
@@ -880,6 +1273,16 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
+[[package]]
+name = "bytes-utils"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35"
+dependencies = [
+ "bytes",
+ "either",
+]
+
[[package]]
name = "bzip2"
version = "0.4.4"
@@ -1313,6 +1716,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff"
+[[package]]
+name = "crc32c"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74"
+dependencies = [
+ "rustc_version 0.4.0",
+]
+
[[package]]
name = "crc32fast"
version = "1.3.2"
@@ -1433,6 +1845,28 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
+[[package]]
+name = "crypto-bigint"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef"
+dependencies = [
+ "generic-array",
+ "rand_core 0.6.4",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "crypto-bigint"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28f85c3514d2a6e64160359b45a3918c3b4178bcbf4ae5d03ab2d02e521c479a"
+dependencies = [
+ "rand_core 0.6.4",
+ "subtle",
+]
+
[[package]]
name = "crypto-common"
version = "0.1.6"
@@ -1704,6 +2138,16 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49"
+[[package]]
+name = "der"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de"
+dependencies = [
+ "const-oid",
+ "zeroize",
+]
+
[[package]]
name = "der"
version = "0.7.8"
@@ -1890,6 +2334,18 @@ version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653"
+[[package]]
+name = "ecdsa"
+version = "0.14.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c"
+dependencies = [
+ "der 0.6.1",
+ "elliptic-curve",
+ "rfc6979",
+ "signature 1.6.4",
+]
+
[[package]]
name = "edge_test_file_write_on_full_disk"
version = "0.0.0"
@@ -1906,6 +2362,26 @@ version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
+[[package]]
+name = "elliptic-curve"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3"
+dependencies = [
+ "base16ct",
+ "crypto-bigint 0.4.9",
+ "der 0.6.1",
+ "digest",
+ "ff",
+ "generic-array",
+ "group",
+ "pkcs8 0.9.0",
+ "rand_core 0.6.4",
+ "sec1",
+ "subtle",
+ "zeroize",
+]
+
[[package]]
name = "encoding_rs"
version = "0.8.33"
@@ -2072,6 +2548,16 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
+[[package]]
+name = "ff"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160"
+dependencies = [
+ "rand_core 0.6.4",
+ "subtle",
+]
+
[[package]]
name = "finl_unicode"
version = "1.2.0"
@@ -2474,6 +2960,17 @@ dependencies = [
"smallvec",
]
+[[package]]
+name = "group"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7"
+dependencies = [
+ "ff",
+ "rand_core 0.6.4",
+ "subtle",
+]
+
[[package]]
name = "h2"
version = "0.3.21"
@@ -2758,7 +3255,9 @@ dependencies = [
"futures-util",
"http",
"hyper",
- "rustls 0.21.7",
+ "log",
+ "rustls 0.21.9",
+ "rustls-native-certs",
"tokio",
"tokio-rustls",
]
@@ -3684,7 +4183,7 @@ dependencies = [
"percent-encoding",
"rand 0.8.5",
"rustc_version_runtime",
- "rustls 0.21.7",
+ "rustls 0.21.9",
"rustls-pemfile",
"serde",
"serde_bytes",
@@ -4261,6 +4760,21 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "opendal-benchmark-vs-s3"
+version = "0.0.0"
+dependencies = [
+ "aws-config",
+ "aws-credential-types",
+ "aws-sdk-s3",
+ "criterion",
+ "dotenvy",
+ "opendal",
+ "rand 0.8.5",
+ "tokio",
+ "uuid",
+]
+
[[package]]
name = "opendal-c"
version = "0.42.0"
@@ -4570,12 +5084,29 @@ version = "6.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1"
+[[package]]
+name = "outref"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a"
+
[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
+[[package]]
+name = "p256"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594"
+dependencies = [
+ "ecdsa",
+ "elliptic-curve",
+ "sha2",
+]
+
[[package]]
name = "panic-message"
version = "0.3.0"
@@ -4792,9 +5323,19 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
dependencies = [
- "der",
- "pkcs8",
- "spki",
+ "der 0.7.8",
+ "pkcs8 0.10.2",
+ "spki 0.7.2",
+]
+
+[[package]]
+name = "pkcs8"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba"
+dependencies = [
+ "der 0.6.1",
+ "spki 0.6.0",
]
[[package]]
@@ -4803,8 +5344,8 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
dependencies = [
- "der",
- "spki",
+ "der 0.7.8",
+ "spki 0.7.2",
]
[[package]]
@@ -5566,7 +6107,7 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"rand 0.8.5",
- "rustls 0.21.7",
+ "rustls 0.21.9",
"rustls-native-certs",
"ryu",
"sha1_smol",
@@ -5728,7 +6269,7 @@ dependencies = [
"once_cell",
"percent-encoding",
"pin-project-lite",
- "rustls 0.21.7",
+ "rustls 0.21.9",
"rustls-native-certs",
"rustls-pemfile",
"serde",
@@ -5765,6 +6306,17 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0"
+[[package]]
+name = "rfc6979"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb"
+dependencies = [
+ "crypto-bigint 0.4.9",
+ "hmac",
+ "zeroize",
+]
+
[[package]]
name = "ring"
version = "0.16.20"
@@ -5846,10 +6398,10 @@ dependencies = [
"num-iter",
"num-traits",
"pkcs1",
- "pkcs8",
+ "pkcs8 0.10.2",
"rand_core 0.6.4",
- "signature",
- "spki",
+ "signature 2.1.0",
+ "spki 0.7.2",
"subtle",
"zeroize",
]
@@ -6011,12 +6563,12 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.21.7"
+version = "0.21.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8"
+checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9"
dependencies = [
"log",
- "ring 0.16.20",
+ "ring 0.17.5",
"rustls-webpki",
"sct 0.7.0",
]
@@ -6044,12 +6596,12 @@ dependencies = [
[[package]]
name = "rustls-webpki"
-version = "0.101.6"
+version = "0.101.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe"
+checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
dependencies = [
- "ring 0.16.20",
- "untrusted 0.7.1",
+ "ring 0.17.5",
+ "untrusted 0.9.0",
]
[[package]]
@@ -6141,6 +6693,20 @@ version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
+[[package]]
+name = "sec1"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928"
+dependencies = [
+ "base16ct",
+ "der 0.6.1",
+ "generic-array",
+ "pkcs8 0.9.0",
+ "subtle",
+ "zeroize",
+]
+
[[package]]
name = "security-framework"
version = "2.9.2"
@@ -6368,6 +6934,16 @@ dependencies = [
"libc",
]
+[[package]]
+name = "signature"
+version = "1.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c"
+dependencies = [
+ "digest",
+ "rand_core 0.6.4",
+]
+
[[package]]
name = "signature"
version = "2.1.0"
@@ -6511,6 +7087,16 @@ dependencies = [
"lock_api",
]
+[[package]]
+name = "spki"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b"
+dependencies = [
+ "base64ct",
+ "der 0.6.1",
+]
+
[[package]]
name = "spki"
version = "0.7.2"
@@ -6518,7 +7104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a"
dependencies = [
"base64ct",
- "der",
+ "der 0.7.8",
]
[[package]]
@@ -6978,7 +7564,7 @@ version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
dependencies = [
- "rustls 0.21.7",
+ "rustls 0.21.9",
"tokio",
]
@@ -7126,7 +7712,7 @@ dependencies = [
"percent-encoding",
"pin-project",
"prost 0.12.1",
- "rustls 0.21.7",
+ "rustls 0.21.9",
"rustls-pemfile",
"tokio",
"tokio-rustls",
@@ -7506,7 +8092,7 @@ dependencies = [
"log",
"native-tls",
"once_cell",
- "rustls 0.21.7",
+ "rustls 0.21.9",
"rustls-webpki",
"url",
"webpki-roots 0.25.2",
@@ -7581,6 +8167,12 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+[[package]]
+name = "vsimd"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
+
[[package]]
name = "wait-timeout"
version = "0.2.0"
@@ -8047,6 +8639,12 @@ version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fcb9cbac069e033553e8bb871be2fbdffcab578eb25bd0f7c508cedc6dcd75a"
+[[package]]
+name = "xmlparser"
+version = "0.13.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
+
[[package]]
name = "xmltree"
version = "0.10.3"
diff --git a/Cargo.toml b/Cargo.toml
index 6ec1b15dd..9db84f398 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -22,7 +22,7 @@ members = [
"core",
"core/fuzz",
"core/edge/*",
- "core/benches/vs_fs",
+ "core/benches/vs_*",
"bindings/c",
"bindings/nodejs",
diff --git a/core/benches/vs_s3/Cargo.toml b/core/benches/vs_s3/Cargo.toml
new file mode 100644
index 000000000..db6a71fe2
--- /dev/null
+++ b/core/benches/vs_s3/Cargo.toml
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "opendal-benchmark-vs-s3"
+description = "OpenDAL Benchmark vs s3"
+version = "0.0.0"
+publish = false
+
+authors.workspace = true
+edition.workspace = true
+homepage.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+opendal = { path = "../..", features = ["tests"] }
+tokio = { version = "1", features = ["full"] }
+uuid = { version = "1", features = ["v4"] }
+criterion = { version = "0.4", features = ["async", "async_tokio"] }
+rand = "0.8"
+aws-sdk-s3 = "0.38"
+aws-config = { version = "0.101.0", features = ["behavior-version-latest"] }
+dotenvy = "0.15"
+aws-credential-types = { version = "0.101.0", features = ["hardcoded-credentials"] }
diff --git a/core/benches/vs_s3/README.md b/core/benches/vs_s3/README.md
new file mode 100644
index 000000000..f79c57a62
--- /dev/null
+++ b/core/benches/vs_s3/README.md
@@ -0,0 +1,40 @@
+# OpenDAL Benchmark VS AWS SDK S3
+
+This benchmark compares the performance of OpenDAL with the performance of the `aws_sdk_s3`.
+
+## Goal
+
+We expect OpenDAL to match `aws_sdk_s3` in speed: the throughput of OpenDAL should be within a `5%` range of `aws_sdk_s3`.
+
+## Usage
+
+For test: `cargo run`
+
+```shell
+Testing read/aws_s3_sdk_collect
+Success
+Testing read/aws_s3_sdk_into_async_read
+Success
+Testing read/opendal_s3
+Success
+Testing read/opendal_s3_with_range
+Success
+```
+
+For bench: `cargo run --release -- --bench`
+
+```shell
+read/aws_s3_sdk_collect time: [50.007 ms 50.265 ms 50.527 ms]
+ thrpt: [316.66 MiB/s 318.31 MiB/s 319.96 MiB/s]
+
+read/aws_s3_sdk_into_async_read
+ time: [11.308 ms 11.675 ms 12.046 ms]
+ thrpt: [1.2971 GiB/s 1.3384 GiB/s 1.3818 GiB/s]
+
+read/opendal_s3 time: [9.5761 ms 9.8121 ms 10.050 ms]
+ thrpt: [1.5548 GiB/s 1.5924 GiB/s 1.6317 GiB/s]
+
+read/opendal_s3_with_range
+ time: [8.9999 ms 9.2246 ms 9.4526 ms]
+ thrpt: [1.6530 GiB/s 1.6938 GiB/s 1.7361 GiB/s]
+```
diff --git a/core/benches/vs_s3/src/main.rs b/core/benches/vs_s3/src/main.rs
new file mode 100644
index 000000000..f7f5c9726
--- /dev/null
+++ b/core/benches/vs_s3/src/main.rs
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use aws_config::{BehaviorVersion, Region};
+use aws_credential_types::Credentials;
+use criterion::Criterion;
+use opendal::raw::tests::TEST_RUNTIME;
+use opendal::services;
+use opendal::Operator;
+use rand::prelude::*;
+use std::env;
+use tokio::io::AsyncReadExt;
+
+fn main() {
+ let _ = dotenvy::dotenv();
+
+ let endpoint = env::var("OPENDAL_S3_ENDPOINT").unwrap();
+ let access_key = env::var("OPENDAL_S3_ACCESS_KEY_ID").unwrap();
+ let secret_key = env::var("OPENDAL_S3_SECRET_ACCESS_KEY").unwrap();
+ let bucket = env::var("OPENDAL_S3_BUCKET").unwrap();
+ let region = env::var("OPENDAL_S3_REGION").unwrap();
+
+ // Init OpenDAL Operator.
+ let mut cfg = services::S3::default();
+ cfg.endpoint(&endpoint);
+ cfg.access_key_id(&access_key);
+ cfg.secret_access_key(&secret_key);
+ cfg.bucket(&bucket);
+ cfg.region(®ion);
+ let op = Operator::new(cfg).unwrap().finish();
+
+ // Init AWS S3 SDK.
+ let mut config_loader = aws_config::defaults(BehaviorVersion::latest());
+ config_loader = config_loader.endpoint_url(&endpoint);
+ config_loader = config_loader.region(Region::new(region.to_string()));
+ config_loader =
+ config_loader.credentials_provider(Credentials::from_keys(&access_key, &secret_key, None));
+ let config = TEST_RUNTIME.block_on(config_loader.load());
+ let s3_client = aws_sdk_s3::Client::new(&config);
+
+ let mut c = Criterion::default().configure_from_args();
+ bench_read(&mut c, op, s3_client, &bucket);
+
+ c.final_summary();
+}
+
+fn bench_read(c: &mut Criterion, op: Operator, s3_client: aws_sdk_s3::Client, bucket: &str) {
+ let mut group = c.benchmark_group("read");
+ group.throughput(criterion::Throughput::Bytes(16 * 1024 * 1024));
+
+ let path = TEST_RUNTIME.block_on(prepare(op.clone()));
+
+ group.bench_function("aws_s3_sdk_collect", |b| {
+ b.to_async(&*TEST_RUNTIME).iter(|| async {
+ let _: Vec<u8> = s3_client
+ .get_object()
+ .bucket(bucket)
+ .key(&path)
+ .send()
+ .await
+ .unwrap()
+ .body
+ .collect()
+ .await
+ .unwrap()
+ .to_vec();
+ });
+ });
+ group.bench_function("aws_s3_sdk_into_async_read", |b| {
+ b.to_async(&*TEST_RUNTIME).iter(|| async {
+ let mut r = s3_client
+ .get_object()
+ .bucket(bucket)
+ .key(&path)
+ .send()
+ .await
+ .unwrap()
+ .body
+ .into_async_read();
+ let mut bs = Vec::new();
+ let _ = r.read_to_end(&mut bs).await.unwrap();
+ });
+ });
+ group.bench_function("opendal_s3", |b| {
+ b.to_async(&*TEST_RUNTIME).iter(|| async {
+ let _: Vec<u8> = op.read(&path).await.unwrap();
+ });
+ });
+ group.bench_function("opendal_s3_with_range", |b| {
+ b.to_async(&*TEST_RUNTIME).iter(|| async {
+ let _: Vec<u8> = op
+ .read_with(&path)
+ .range(0..16 * 1024 * 1024)
+ .await
+ .unwrap();
+ });
+ });
+
+ group.finish()
+}
+
+async fn prepare(op: Operator) -> String {
+ let mut rng = thread_rng();
+ let mut content = vec![0; 16 * 1024 * 1024];
+ rng.fill_bytes(&mut content);
+
+ let name = uuid::Uuid::new_v4().to_string();
+ op.write(&name, content).await.unwrap();
+
+ name
+}
diff --git a/core/src/raw/http_util/body.rs b/core/src/raw/http_util/body.rs
index 8b7d5da5a..fec20f9ee 100644
--- a/core/src/raw/http_util/body.rs
+++ b/core/src/raw/http_util/body.rs
@@ -159,13 +159,19 @@ impl oio::Read for IncomingAsyncBody {
return Poll::Ready(Ok(0));
}
- // We must get a valid bytes from underlying stream
- let mut bs = loop {
- match ready!(self.poll_next(cx)) {
- Some(Ok(bs)) if bs.is_empty() => continue,
- Some(Ok(bs)) => break bs,
- Some(Err(err)) => return Poll::Ready(Err(err)),
- None => return Poll::Ready(Ok(0)),
+ // Avoid extra poll of next if we already have chunks.
+ let mut bs = if let Some(chunk) = self.chunk.take() {
+ chunk
+ } else {
+ loop {
+ match ready!(self.poll_next(cx)) {
+ // It's possible for underlying stream to return empty bytes, we should continue
+ // to fetch next one.
+ Some(Ok(bs)) if bs.is_empty() => continue,
+ Some(Ok(bs)) => break bs,
+ Some(Err(err)) => return Poll::Ready(Err(err)),
+ None => return Poll::Ready(Ok(0)),
+ }
}
};
diff --git a/core/src/raw/oio/read/api.rs b/core/src/raw/oio/read/api.rs
index 03a362c6d..cbb8e20d1 100644
--- a/core/src/raw/oio/read/api.rs
+++ b/core/src/raw/oio/read/api.rs
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-use std::cmp;
use std::fmt::Display;
use std::fmt::Formatter;
use std::io;
@@ -204,14 +203,7 @@ pub trait ReadExt: Read {
/// Build a future for `read_to_end`.
fn read_to_end<'a>(&'a mut self, buf: &'a mut Vec<u8>) -> ReadToEndFuture<'a, Self> {
- let start = buf.len();
- ReadToEndFuture {
- reader: self,
- buf,
- start,
- length: start,
- next: MIN_READ_TO_END_GROW_SIZE,
- }
+ ReadToEndFuture { reader: self, buf }
}
}
@@ -271,19 +263,11 @@ where
}
}
-/// The MIN read to end grow size.
-const MIN_READ_TO_END_GROW_SIZE: usize = 8 * 1024;
-/// The MAX read to end grow size.
-const MAX_READ_TO_END_GROW_SIZE: usize = 4 * 1024 * 1024;
-
/// Make this future `!Unpin` for compatibility with async trait methods.
#[pin_project(!Unpin)]
pub struct ReadToEndFuture<'a, R: Read + Unpin + ?Sized> {
reader: &'a mut R,
buf: &'a mut Vec<u8>,
- start: usize,
- length: usize,
- next: usize,
}
impl<R> Future for ReadToEndFuture<'_, R>
@@ -294,43 +278,50 @@ where
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<usize>> {
let this = self.project();
+ let start_len = this.buf.len();
+ let start_cap = this.buf.capacity();
loop {
- if this.buf.capacity() == *this.length {
- this.buf.reserve(*this.next);
- // # Safety
- //
- // We make sure that the length of buf is maintained correctly.
- #[allow(clippy::uninit_vec)]
- unsafe {
- this.buf.set_len(this.buf.capacity());
- }
+ if this.buf.len() == this.buf.capacity() {
+ this.buf.reserve(32); // buf is full, need more space
}
- let buf = &mut this.buf[*this.length..];
- match ready!(this.reader.poll_read(cx, buf)) {
+ let spare = this.buf.spare_capacity_mut();
+ let mut read_buf: ReadBuf = ReadBuf::uninit(spare);
+
+ // SAFETY: These bytes were initialized but not filled in the previous loop
+ unsafe {
+ read_buf.assume_init(read_buf.capacity());
+ }
+
+ match ready!(this.reader.poll_read(cx, read_buf.initialize_unfilled())) {
Ok(0) => {
- unsafe {
- this.buf.set_len(*this.length);
- }
- return Poll::Ready(Ok(*this.length - *this.start));
+ return Poll::Ready(Ok(this.buf.len() - start_len));
}
Ok(n) => {
- *this.next = if n >= *this.next {
- cmp::min((*this.next).saturating_mul(2), MAX_READ_TO_END_GROW_SIZE)
- } else if n >= *this.next / 2 {
- *this.next
- } else {
- cmp::max((*this.next).saturating_div(2), MIN_READ_TO_END_GROW_SIZE)
- };
- // We can't allow bogus values from read. If it is too large, the returned vec could have its length
- // set past its capacity, or if it overflows the vec could be shortened which could create an invalid
- // string if this is called via read_to_string.
- assert!(n <= buf.len());
- *this.length += n;
+ // SAFETY: Read API makes sure that returning `n` is correct.
+ unsafe {
+ this.buf.set_len(this.buf.len() + n);
+ }
}
Err(e) => return Poll::Ready(Err(e)),
}
+
+ // The buffer might be an exact fit. Let's read into a probe buffer
+ // and see if it returns `Ok(0)`. If so, we've avoided an
+ // unnecessary doubling of the capacity. But if not, append the
+ // probe buffer to the primary buffer and let its capacity grow.
+ if this.buf.len() == this.buf.capacity() && this.buf.capacity() == start_cap {
+ let mut probe = [0u8; 32];
+
+ match ready!(this.reader.poll_read(cx, &mut probe)) {
+ Ok(0) => return Poll::Ready(Ok(this.buf.len() - start_len)),
+ Ok(n) => {
+ this.buf.extend_from_slice(&probe[..n]);
+ }
+ Err(e) => return Poll::Ready(Err(e)),
+ }
+ }
}
}
}
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index 0d24728f5..e1de7cc74 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -367,8 +367,22 @@ impl Operator {
.with_context("path", &path));
}
+ let range = args.range();
+ let size_hint = match range.size() {
+ Some(v) => v,
+ None => {
+ let mut size = inner
+ .stat(&path, OpStat::default())
+ .await?
+ .into_metadata()
+ .content_length();
+ size -= range.offset().unwrap_or(0);
+ size
+ }
+ };
+
let (_, mut s) = inner.read(&path, args).await?;
- let mut buf = Vec::new();
+ let mut buf = Vec::with_capacity(size_hint as usize);
s.read_to_end(&mut buf).await?;
Ok(buf)