You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/11/18 17:20:14 UTC
(incubator-opendal) branch benchmark-s3 created (now dd9908934)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a change to branch benchmark-s3
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
at dd9908934 feat: Add benchmark vs aws sdk s3
This branch includes the following new commits:
new dd9908934 feat: Add benchmark vs aws sdk s3
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-opendal) 01/01: feat: Add benchmark vs aws sdk s3
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch benchmark-s3
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit dd99089341451f8ce344558eb3880289ccc96650
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Sun Nov 19 01:19:53 2023 +0800
feat: Add benchmark vs aws sdk s3
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
Cargo.lock | 644 ++++++++++++++++++++++++++++++++++--
Cargo.toml | 2 +-
core/benches/vs_s3/Cargo.toml | 40 +++
core/benches/vs_s3/README.md | 40 +++
core/benches/vs_s3/src/main.rs | 125 +++++++
core/src/raw/http_util/body.rs | 20 +-
core/src/raw/oio/read/api.rs | 79 ++---
core/src/types/operator/operator.rs | 16 +-
8 files changed, 890 insertions(+), 76 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 662052bd2..f2a7ddfde 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -497,6 +497,383 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5b3469636cdf8543cceab175efca534471f36eee12fb8374aba00eb5e7e7f8a"
+[[package]]
+name = "aws-config"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "84f9625b71b3ee4adbfbca369c6680d156e316ed86d2c7199a2a134563917414"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-runtime",
+ "aws-sdk-sso",
+ "aws-sdk-ssooidc",
+ "aws-sdk-sts",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "fastrand 2.0.1",
+ "hex",
+ "http",
+ "hyper",
+ "ring 0.17.5",
+ "time",
+ "tokio",
+ "tracing",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-credential-types"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5924466398ac76ffd411d297b9d516dcebb0577f7344c0c15fd8e8e04d9c7895"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-http"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb9a3aa335a105a00975c971f1dad403c3175f2a210d98f39345c6af53923912"
+dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "http",
+ "http-body",
+ "pin-project-lite",
+ "tracing",
+]
+
+[[package]]
+name = "aws-runtime"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b75844ecbdf3dc5e0f5ac5fd1088fb1623849990ea9445d2826258ce63be4de5"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-sigv4",
+ "aws-smithy-async",
+ "aws-smithy-eventstream",
+ "aws-smithy-http",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "fastrand 2.0.1",
+ "http",
+ "percent-encoding",
+ "tracing",
+ "uuid",
+]
+
+[[package]]
+name = "aws-sdk-s3"
+version = "0.38.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39a2ccbd49e784c36d0f0596da681275496f679e249059cae3fa92dd3c2c72e1"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-runtime",
+ "aws-sigv4",
+ "aws-smithy-async",
+ "aws-smithy-checksums",
+ "aws-smithy-eventstream",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-smithy-xml",
+ "aws-types",
+ "bytes",
+ "http",
+ "http-body",
+ "once_cell",
+ "percent-encoding",
+ "regex",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "aws-sdk-sso"
+version = "0.38.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c870aa95e1e85f837f74af2cc937b3f8e72e2315a89e524265875843655b4d47"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "http",
+ "regex",
+ "tracing",
+]
+
+[[package]]
+name = "aws-sdk-ssooidc"
+version = "0.38.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "107ee812e46f9120e68d48bf985d2f2a538315bd8be8a3e54db619250cc4c95e"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "http",
+ "regex",
+ "tracing",
+]
+
+[[package]]
+name = "aws-sdk-sts"
+version = "0.38.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4e3958c43a78f6c3822e62009a35802af5cc7c120fbe8e60b98565604569aae"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-query",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-smithy-xml",
+ "aws-types",
+ "http",
+ "regex",
+ "tracing",
+]
+
+[[package]]
+name = "aws-sigv4"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06130e3686db3c5ae2fc44b3516fffe6b4d4eccebe09bd8ccc4067f3c9c183fb"
+dependencies = [
+ "aws-credential-types",
+ "aws-smithy-eventstream",
+ "aws-smithy-http",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "bytes",
+ "crypto-bigint 0.5.4",
+ "form_urlencoded",
+ "hex",
+ "hmac",
+ "http",
+ "once_cell",
+ "p256",
+ "percent-encoding",
+ "regex",
+ "ring 0.17.5",
+ "sha2",
+ "subtle",
+ "time",
+ "tracing",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-smithy-async"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d787b7e07925b450bed90d9d29ac8e57006c9c2ac907151d175ac0e376bfee0e"
+dependencies = [
+ "futures-util",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "aws-smithy-checksums"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d293ef76a982c573f7384e30c28f9a2472f8dd5f4ce5abcceb0e909a15098e8e"
+dependencies = [
+ "aws-smithy-http",
+ "aws-smithy-types",
+ "bytes",
+ "crc32c",
+ "crc32fast",
+ "hex",
+ "http",
+ "http-body",
+ "md-5",
+ "pin-project-lite",
+ "sha1",
+ "sha2",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-eventstream"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1d2c1844aee465dc023924dbe19d730b116eaf3587d7c2a9b4a41f9c4e980ee"
+dependencies = [
+ "aws-smithy-types",
+ "bytes",
+ "crc32fast",
+]
+
+[[package]]
+name = "aws-smithy-http"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96daaad925331c72449423574fdc72b54af780d5a23ace3c0a6ad0ccbf378715"
+dependencies = [
+ "aws-smithy-eventstream",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "bytes",
+ "bytes-utils",
+ "futures-core",
+ "http",
+ "http-body",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "pin-utils",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-json"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ff985bee3fe21046dc501fadc1d04a1161977c55a0cbbccd9b111c18206aa64"
+dependencies = [
+ "aws-smithy-types",
+]
+
+[[package]]
+name = "aws-smithy-query"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb4006503693766d34717efc5f58325062845fce26a683a71b70f23156d72e67"
+dependencies = [
+ "aws-smithy-types",
+ "urlencoding",
+]
+
+[[package]]
+name = "aws-smithy-runtime"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d28af854558601b4202a4273b9720aebe43d73e472143e6056f16e3bd90bc837"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "bytes",
+ "fastrand 2.0.1",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-rustls",
+ "once_cell",
+ "pin-project-lite",
+ "pin-utils",
+ "rustls 0.21.9",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "aws-smithy-runtime-api"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1c68e17e754b86da350b43add38294189121a880e9c3fb454f83ff7044f5257"
+dependencies = [
+ "aws-smithy-async",
+ "aws-smithy-types",
+ "bytes",
+ "http",
+ "pin-project-lite",
+ "tokio",
+ "tracing",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-smithy-types"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d97b978d8a351ea5744206ecc643a1d3806628680e9f151b4d6b7a76fec1596f"
+dependencies = [
+ "base64-simd",
+ "bytes",
+ "bytes-utils",
+ "futures-core",
+ "http",
+ "http-body",
+ "itoa",
+ "num-integer",
+ "pin-project-lite",
+ "pin-utils",
+ "ryu",
+ "serde",
+ "time",
+ "tokio",
+ "tokio-util",
+]
+
+[[package]]
+name = "aws-smithy-xml"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97500a0d0884b9576e65076075f81d899cfbb84f7db5ca1dd317f0582204e528"
+dependencies = [
+ "xmlparser",
+]
+
+[[package]]
+name = "aws-types"
+version = "0.101.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61065f0c6070cb0f9aaddfa614605fb1049908481da71ba5b39b2ffca12f57e4"
+dependencies = [
+ "aws-credential-types",
+ "aws-smithy-async",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "http",
+ "rustc_version 0.4.0",
+ "tracing",
+]
+
[[package]]
name = "axum"
version = "0.6.20"
@@ -573,6 +950,12 @@ dependencies = [
"rustc-demangle",
]
+[[package]]
+name = "base16ct"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce"
+
[[package]]
name = "base64"
version = "0.13.1"
@@ -585,6 +968,16 @@ version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2"
+[[package]]
+name = "base64-simd"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195"
+dependencies = [
+ "outref",
+ "vsimd",
+]
+
[[package]]
name = "base64ct"
version = "1.6.0"
@@ -880,6 +1273,16 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
+[[package]]
+name = "bytes-utils"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35"
+dependencies = [
+ "bytes",
+ "either",
+]
+
[[package]]
name = "bzip2"
version = "0.4.4"
@@ -1313,6 +1716,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff"
+[[package]]
+name = "crc32c"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74"
+dependencies = [
+ "rustc_version 0.4.0",
+]
+
[[package]]
name = "crc32fast"
version = "1.3.2"
@@ -1433,6 +1845,28 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
+[[package]]
+name = "crypto-bigint"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef"
+dependencies = [
+ "generic-array",
+ "rand_core 0.6.4",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "crypto-bigint"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28f85c3514d2a6e64160359b45a3918c3b4178bcbf4ae5d03ab2d02e521c479a"
+dependencies = [
+ "rand_core 0.6.4",
+ "subtle",
+]
+
[[package]]
name = "crypto-common"
version = "0.1.6"
@@ -1704,6 +2138,16 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49"
+[[package]]
+name = "der"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de"
+dependencies = [
+ "const-oid",
+ "zeroize",
+]
+
[[package]]
name = "der"
version = "0.7.8"
@@ -1890,6 +2334,18 @@ version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653"
+[[package]]
+name = "ecdsa"
+version = "0.14.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c"
+dependencies = [
+ "der 0.6.1",
+ "elliptic-curve",
+ "rfc6979",
+ "signature 1.6.4",
+]
+
[[package]]
name = "edge_test_file_write_on_full_disk"
version = "0.0.0"
@@ -1906,6 +2362,26 @@ version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
+[[package]]
+name = "elliptic-curve"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3"
+dependencies = [
+ "base16ct",
+ "crypto-bigint 0.4.9",
+ "der 0.6.1",
+ "digest",
+ "ff",
+ "generic-array",
+ "group",
+ "pkcs8 0.9.0",
+ "rand_core 0.6.4",
+ "sec1",
+ "subtle",
+ "zeroize",
+]
+
[[package]]
name = "encoding_rs"
version = "0.8.33"
@@ -2072,6 +2548,16 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
+[[package]]
+name = "ff"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160"
+dependencies = [
+ "rand_core 0.6.4",
+ "subtle",
+]
+
[[package]]
name = "finl_unicode"
version = "1.2.0"
@@ -2474,6 +2960,17 @@ dependencies = [
"smallvec",
]
+[[package]]
+name = "group"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7"
+dependencies = [
+ "ff",
+ "rand_core 0.6.4",
+ "subtle",
+]
+
[[package]]
name = "h2"
version = "0.3.21"
@@ -2758,7 +3255,9 @@ dependencies = [
"futures-util",
"http",
"hyper",
- "rustls 0.21.7",
+ "log",
+ "rustls 0.21.9",
+ "rustls-native-certs",
"tokio",
"tokio-rustls",
]
@@ -3684,7 +4183,7 @@ dependencies = [
"percent-encoding",
"rand 0.8.5",
"rustc_version_runtime",
- "rustls 0.21.7",
+ "rustls 0.21.9",
"rustls-pemfile",
"serde",
"serde_bytes",
@@ -4261,6 +4760,21 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "opendal-benchmark-vs-s3"
+version = "0.0.0"
+dependencies = [
+ "aws-config",
+ "aws-credential-types",
+ "aws-sdk-s3",
+ "criterion",
+ "dotenvy",
+ "opendal",
+ "rand 0.8.5",
+ "tokio",
+ "uuid",
+]
+
[[package]]
name = "opendal-c"
version = "0.42.0"
@@ -4570,12 +5084,29 @@ version = "6.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1"
+[[package]]
+name = "outref"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a"
+
[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
+[[package]]
+name = "p256"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594"
+dependencies = [
+ "ecdsa",
+ "elliptic-curve",
+ "sha2",
+]
+
[[package]]
name = "panic-message"
version = "0.3.0"
@@ -4792,9 +5323,19 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
dependencies = [
- "der",
- "pkcs8",
- "spki",
+ "der 0.7.8",
+ "pkcs8 0.10.2",
+ "spki 0.7.2",
+]
+
+[[package]]
+name = "pkcs8"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba"
+dependencies = [
+ "der 0.6.1",
+ "spki 0.6.0",
]
[[package]]
@@ -4803,8 +5344,8 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
dependencies = [
- "der",
- "spki",
+ "der 0.7.8",
+ "spki 0.7.2",
]
[[package]]
@@ -5566,7 +6107,7 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"rand 0.8.5",
- "rustls 0.21.7",
+ "rustls 0.21.9",
"rustls-native-certs",
"ryu",
"sha1_smol",
@@ -5728,7 +6269,7 @@ dependencies = [
"once_cell",
"percent-encoding",
"pin-project-lite",
- "rustls 0.21.7",
+ "rustls 0.21.9",
"rustls-native-certs",
"rustls-pemfile",
"serde",
@@ -5765,6 +6306,17 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0"
+[[package]]
+name = "rfc6979"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb"
+dependencies = [
+ "crypto-bigint 0.4.9",
+ "hmac",
+ "zeroize",
+]
+
[[package]]
name = "ring"
version = "0.16.20"
@@ -5846,10 +6398,10 @@ dependencies = [
"num-iter",
"num-traits",
"pkcs1",
- "pkcs8",
+ "pkcs8 0.10.2",
"rand_core 0.6.4",
- "signature",
- "spki",
+ "signature 2.1.0",
+ "spki 0.7.2",
"subtle",
"zeroize",
]
@@ -6011,12 +6563,12 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.21.7"
+version = "0.21.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8"
+checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9"
dependencies = [
"log",
- "ring 0.16.20",
+ "ring 0.17.5",
"rustls-webpki",
"sct 0.7.0",
]
@@ -6044,12 +6596,12 @@ dependencies = [
[[package]]
name = "rustls-webpki"
-version = "0.101.6"
+version = "0.101.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe"
+checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
dependencies = [
- "ring 0.16.20",
- "untrusted 0.7.1",
+ "ring 0.17.5",
+ "untrusted 0.9.0",
]
[[package]]
@@ -6141,6 +6693,20 @@ version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
+[[package]]
+name = "sec1"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928"
+dependencies = [
+ "base16ct",
+ "der 0.6.1",
+ "generic-array",
+ "pkcs8 0.9.0",
+ "subtle",
+ "zeroize",
+]
+
[[package]]
name = "security-framework"
version = "2.9.2"
@@ -6368,6 +6934,16 @@ dependencies = [
"libc",
]
+[[package]]
+name = "signature"
+version = "1.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c"
+dependencies = [
+ "digest",
+ "rand_core 0.6.4",
+]
+
[[package]]
name = "signature"
version = "2.1.0"
@@ -6511,6 +7087,16 @@ dependencies = [
"lock_api",
]
+[[package]]
+name = "spki"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b"
+dependencies = [
+ "base64ct",
+ "der 0.6.1",
+]
+
[[package]]
name = "spki"
version = "0.7.2"
@@ -6518,7 +7104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a"
dependencies = [
"base64ct",
- "der",
+ "der 0.7.8",
]
[[package]]
@@ -6978,7 +7564,7 @@ version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
dependencies = [
- "rustls 0.21.7",
+ "rustls 0.21.9",
"tokio",
]
@@ -7126,7 +7712,7 @@ dependencies = [
"percent-encoding",
"pin-project",
"prost 0.12.1",
- "rustls 0.21.7",
+ "rustls 0.21.9",
"rustls-pemfile",
"tokio",
"tokio-rustls",
@@ -7506,7 +8092,7 @@ dependencies = [
"log",
"native-tls",
"once_cell",
- "rustls 0.21.7",
+ "rustls 0.21.9",
"rustls-webpki",
"url",
"webpki-roots 0.25.2",
@@ -7581,6 +8167,12 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+[[package]]
+name = "vsimd"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
+
[[package]]
name = "wait-timeout"
version = "0.2.0"
@@ -8047,6 +8639,12 @@ version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fcb9cbac069e033553e8bb871be2fbdffcab578eb25bd0f7c508cedc6dcd75a"
+[[package]]
+name = "xmlparser"
+version = "0.13.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
+
[[package]]
name = "xmltree"
version = "0.10.3"
diff --git a/Cargo.toml b/Cargo.toml
index 6ec1b15dd..9db84f398 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -22,7 +22,7 @@ members = [
"core",
"core/fuzz",
"core/edge/*",
- "core/benches/vs_fs",
+ "core/benches/vs_*",
"bindings/c",
"bindings/nodejs",
diff --git a/core/benches/vs_s3/Cargo.toml b/core/benches/vs_s3/Cargo.toml
new file mode 100644
index 000000000..db6a71fe2
--- /dev/null
+++ b/core/benches/vs_s3/Cargo.toml
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "opendal-benchmark-vs-s3"
+description = "OpenDAL Benchmark vs s3"
+version = "0.0.0"
+publish = false
+
+authors.workspace = true
+edition.workspace = true
+homepage.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+opendal = { path = "../..", features = ["tests"] }
+tokio = { version = "1", features = ["full"] }
+uuid = { version = "1", features = ["v4"] }
+criterion = { version = "0.4", features = ["async", "async_tokio"] }
+rand = "0.8"
+aws-sdk-s3 = "0.38"
+aws-config = { version = "0.101.0", features = ["behavior-version-latest"] }
+dotenvy = "0.15"
+aws-credential-types = { version = "0.101.0", features = ["hardcoded-credentials"] }
diff --git a/core/benches/vs_s3/README.md b/core/benches/vs_s3/README.md
new file mode 100644
index 000000000..f79c57a62
--- /dev/null
+++ b/core/benches/vs_s3/README.md
@@ -0,0 +1,40 @@
+# OpenDAL Benchmark VS AWS SDK S3
+
+This benchmark compares the performance of OpenDAL with the performance of the `aws_sdk_s3`.
+
+## Goal
+
+We expect OpenDAL to match `aws_sdk_s3` in speed: the throughput of OpenDAL should be within a `5%` range of `aws_sdk_s3`.
+
+## Usage
+
+For test: `cargo run`
+
+```shell
+Testing read/aws_s3_sdk_collect
+Success
+Testing read/aws_s3_sdk_into_async_read
+Success
+Testing read/opendal_s3
+Success
+Testing read/opendal_s3_with_range
+Success
+```
+
+For bench: `cargo run --release -- --bench`
+
+```shell
+read/aws_s3_sdk_collect time: [50.007 ms 50.265 ms 50.527 ms]
+ thrpt: [316.66 MiB/s 318.31 MiB/s 319.96 MiB/s]
+
+read/aws_s3_sdk_into_async_read
+ time: [11.308 ms 11.675 ms 12.046 ms]
+ thrpt: [1.2971 GiB/s 1.3384 GiB/s 1.3818 GiB/s]
+
+read/opendal_s3 time: [9.5761 ms 9.8121 ms 10.050 ms]
+ thrpt: [1.5548 GiB/s 1.5924 GiB/s 1.6317 GiB/s]
+
+read/opendal_s3_with_range
+ time: [8.9999 ms 9.2246 ms 9.4526 ms]
+ thrpt: [1.6530 GiB/s 1.6938 GiB/s 1.7361 GiB/s]
+```
diff --git a/core/benches/vs_s3/src/main.rs b/core/benches/vs_s3/src/main.rs
new file mode 100644
index 000000000..f7f5c9726
--- /dev/null
+++ b/core/benches/vs_s3/src/main.rs
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use aws_config::{BehaviorVersion, Region};
+use aws_credential_types::Credentials;
+use criterion::Criterion;
+use opendal::raw::tests::TEST_RUNTIME;
+use opendal::services;
+use opendal::Operator;
+use rand::prelude::*;
+use std::env;
+use tokio::io::AsyncReadExt;
+
+fn main() {
+ let _ = dotenvy::dotenv();
+
+ let endpoint = env::var("OPENDAL_S3_ENDPOINT").unwrap();
+ let access_key = env::var("OPENDAL_S3_ACCESS_KEY_ID").unwrap();
+ let secret_key = env::var("OPENDAL_S3_SECRET_ACCESS_KEY").unwrap();
+ let bucket = env::var("OPENDAL_S3_BUCKET").unwrap();
+ let region = env::var("OPENDAL_S3_REGION").unwrap();
+
+ // Init OpenDAL Operator.
+ let mut cfg = services::S3::default();
+ cfg.endpoint(&endpoint);
+ cfg.access_key_id(&access_key);
+ cfg.secret_access_key(&secret_key);
+ cfg.bucket(&bucket);
+ cfg.region(®ion);
+ let op = Operator::new(cfg).unwrap().finish();
+
+ // Init AWS S3 SDK.
+ let mut config_loader = aws_config::defaults(BehaviorVersion::latest());
+ config_loader = config_loader.endpoint_url(&endpoint);
+ config_loader = config_loader.region(Region::new(region.to_string()));
+ config_loader =
+ config_loader.credentials_provider(Credentials::from_keys(&access_key, &secret_key, None));
+ let config = TEST_RUNTIME.block_on(config_loader.load());
+ let s3_client = aws_sdk_s3::Client::new(&config);
+
+ let mut c = Criterion::default().configure_from_args();
+ bench_read(&mut c, op, s3_client, &bucket);
+
+ c.final_summary();
+}
+
+fn bench_read(c: &mut Criterion, op: Operator, s3_client: aws_sdk_s3::Client, bucket: &str) {
+ let mut group = c.benchmark_group("read");
+ group.throughput(criterion::Throughput::Bytes(16 * 1024 * 1024));
+
+ let path = TEST_RUNTIME.block_on(prepare(op.clone()));
+
+ group.bench_function("aws_s3_sdk_collect", |b| {
+ b.to_async(&*TEST_RUNTIME).iter(|| async {
+ let _: Vec<u8> = s3_client
+ .get_object()
+ .bucket(bucket)
+ .key(&path)
+ .send()
+ .await
+ .unwrap()
+ .body
+ .collect()
+ .await
+ .unwrap()
+ .to_vec();
+ });
+ });
+ group.bench_function("aws_s3_sdk_into_async_read", |b| {
+ b.to_async(&*TEST_RUNTIME).iter(|| async {
+ let mut r = s3_client
+ .get_object()
+ .bucket(bucket)
+ .key(&path)
+ .send()
+ .await
+ .unwrap()
+ .body
+ .into_async_read();
+ let mut bs = Vec::new();
+ let _ = r.read_to_end(&mut bs).await.unwrap();
+ });
+ });
+ group.bench_function("opendal_s3", |b| {
+ b.to_async(&*TEST_RUNTIME).iter(|| async {
+ let _: Vec<u8> = op.read(&path).await.unwrap();
+ });
+ });
+ group.bench_function("opendal_s3_with_range", |b| {
+ b.to_async(&*TEST_RUNTIME).iter(|| async {
+ let _: Vec<u8> = op
+ .read_with(&path)
+ .range(0..16 * 1024 * 1024)
+ .await
+ .unwrap();
+ });
+ });
+
+ group.finish()
+}
+
+async fn prepare(op: Operator) -> String {
+ let mut rng = thread_rng();
+ let mut content = vec![0; 16 * 1024 * 1024];
+ rng.fill_bytes(&mut content);
+
+ let name = uuid::Uuid::new_v4().to_string();
+ op.write(&name, content).await.unwrap();
+
+ name
+}
diff --git a/core/src/raw/http_util/body.rs b/core/src/raw/http_util/body.rs
index 8b7d5da5a..fec20f9ee 100644
--- a/core/src/raw/http_util/body.rs
+++ b/core/src/raw/http_util/body.rs
@@ -159,13 +159,19 @@ impl oio::Read for IncomingAsyncBody {
return Poll::Ready(Ok(0));
}
- // We must get a valid bytes from underlying stream
- let mut bs = loop {
- match ready!(self.poll_next(cx)) {
- Some(Ok(bs)) if bs.is_empty() => continue,
- Some(Ok(bs)) => break bs,
- Some(Err(err)) => return Poll::Ready(Err(err)),
- None => return Poll::Ready(Ok(0)),
+ // Avoid extra poll of next if we already have chunks.
+ let mut bs = if let Some(chunk) = self.chunk.take() {
+ chunk
+ } else {
+ loop {
+ match ready!(self.poll_next(cx)) {
+ // It's possible for underlying stream to return empty bytes, we should continue
+ // to fetch next one.
+ Some(Ok(bs)) if bs.is_empty() => continue,
+ Some(Ok(bs)) => break bs,
+ Some(Err(err)) => return Poll::Ready(Err(err)),
+ None => return Poll::Ready(Ok(0)),
+ }
}
};
diff --git a/core/src/raw/oio/read/api.rs b/core/src/raw/oio/read/api.rs
index 03a362c6d..cbb8e20d1 100644
--- a/core/src/raw/oio/read/api.rs
+++ b/core/src/raw/oio/read/api.rs
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-use std::cmp;
use std::fmt::Display;
use std::fmt::Formatter;
use std::io;
@@ -204,14 +203,7 @@ pub trait ReadExt: Read {
/// Build a future for `read_to_end`.
fn read_to_end<'a>(&'a mut self, buf: &'a mut Vec<u8>) -> ReadToEndFuture<'a, Self> {
- let start = buf.len();
- ReadToEndFuture {
- reader: self,
- buf,
- start,
- length: start,
- next: MIN_READ_TO_END_GROW_SIZE,
- }
+ ReadToEndFuture { reader: self, buf }
}
}
@@ -271,19 +263,11 @@ where
}
}
-/// The MIN read to end grow size.
-const MIN_READ_TO_END_GROW_SIZE: usize = 8 * 1024;
-/// The MAX read to end grow size.
-const MAX_READ_TO_END_GROW_SIZE: usize = 4 * 1024 * 1024;
-
/// Make this future `!Unpin` for compatibility with async trait methods.
#[pin_project(!Unpin)]
pub struct ReadToEndFuture<'a, R: Read + Unpin + ?Sized> {
reader: &'a mut R,
buf: &'a mut Vec<u8>,
- start: usize,
- length: usize,
- next: usize,
}
impl<R> Future for ReadToEndFuture<'_, R>
@@ -294,43 +278,50 @@ where
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<usize>> {
let this = self.project();
+ let start_len = this.buf.len();
+ let start_cap = this.buf.capacity();
loop {
- if this.buf.capacity() == *this.length {
- this.buf.reserve(*this.next);
- // # Safety
- //
- // We make sure that the length of buf is maintained correctly.
- #[allow(clippy::uninit_vec)]
- unsafe {
- this.buf.set_len(this.buf.capacity());
- }
+ if this.buf.len() == this.buf.capacity() {
+ this.buf.reserve(32); // buf is full, need more space
}
- let buf = &mut this.buf[*this.length..];
- match ready!(this.reader.poll_read(cx, buf)) {
+ let spare = this.buf.spare_capacity_mut();
+ let mut read_buf: ReadBuf = ReadBuf::uninit(spare);
+
+ // SAFETY: These bytes were initialized but not filled in the previous loop
+ unsafe {
+ read_buf.assume_init(read_buf.capacity());
+ }
+
+ match ready!(this.reader.poll_read(cx, read_buf.initialize_unfilled())) {
Ok(0) => {
- unsafe {
- this.buf.set_len(*this.length);
- }
- return Poll::Ready(Ok(*this.length - *this.start));
+ return Poll::Ready(Ok(this.buf.len() - start_len));
}
Ok(n) => {
- *this.next = if n >= *this.next {
- cmp::min((*this.next).saturating_mul(2), MAX_READ_TO_END_GROW_SIZE)
- } else if n >= *this.next / 2 {
- *this.next
- } else {
- cmp::max((*this.next).saturating_div(2), MIN_READ_TO_END_GROW_SIZE)
- };
- // We can't allow bogus values from read. If it is too large, the returned vec could have its length
- // set past its capacity, or if it overflows the vec could be shortened which could create an invalid
- // string if this is called via read_to_string.
- assert!(n <= buf.len());
- *this.length += n;
+ // SAFETY: Read API makes sure that returning `n` is correct.
+ unsafe {
+ this.buf.set_len(this.buf.len() + n);
+ }
}
Err(e) => return Poll::Ready(Err(e)),
}
+
+ // The buffer might be an exact fit. Let's read into a probe buffer
+ // and see if it returns `Ok(0)`. If so, we've avoided an
+ // unnecessary doubling of the capacity. But if not, append the
+ // probe buffer to the primary buffer and let its capacity grow.
+ if this.buf.len() == this.buf.capacity() && this.buf.capacity() == start_cap {
+ let mut probe = [0u8; 32];
+
+ match ready!(this.reader.poll_read(cx, &mut probe)) {
+ Ok(0) => return Poll::Ready(Ok(this.buf.len() - start_len)),
+ Ok(n) => {
+ this.buf.extend_from_slice(&probe[..n]);
+ }
+ Err(e) => return Poll::Ready(Err(e)),
+ }
+ }
}
}
}
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index 0d24728f5..e1de7cc74 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -367,8 +367,22 @@ impl Operator {
.with_context("path", &path));
}
+ let range = args.range();
+ let size_hint = match range.size() {
+ Some(v) => v,
+ None => {
+ let mut size = inner
+ .stat(&path, OpStat::default())
+ .await?
+ .into_metadata()
+ .content_length();
+ size -= range.offset().unwrap_or(0);
+ size
+ }
+ };
+
let (_, mut s) = inner.read(&path, args).await?;
- let mut buf = Vec::new();
+ let mut buf = Vec::with_capacity(size_hint as usize);
s.read_to_end(&mut buf).await?;
Ok(buf)