You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/05/06 15:43:04 UTC

[incubator-opendal] branch add-s3-api updated (454ffdac -> 95a1af82)

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a change to branch add-s3-api
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git


    from 454ffdac feat(oay): Add list objects v2 support
     add f2e9bde0 feat(services/OSS): Add override_content_disposition support (#2043)
     add d382b0fd feat: add IF_MATCH for http (#2044)
     add be2a3a75 feat(bindings/nodejs): publish support `--provenance` (#2046)
     add 6ce6eadd feat: add IF_MATCH for http HEAD request (#2047)
     add fa1faecc chore: add cache control header for azblob and obs (#2049)
     add ec61c873 fix(bindings/ruby): use rb_sys_env to help find ruby for building (#2051)
     add a32b92ec ci: upgrade typos to 1.14.8 (#2055)
     add 3ab66f5d chore(bindings/C): ignore the formatting of auto-generated opendal.h (#2056)
     add 863d3449 feat: Add capability for operation's variant and args (#2057)
     add 80ab4194 fix: MadsimLayer should be able to built without cfg (#2059)
     add fe92691d docs(bindings/python): ipynb examples for users (#2061)
     add f4961f3f fix(services/s3): Ignore prefix if it's empty (#2064)
     add 40a0d04b refactor(raw/http_util): Add url in error context (#2066)
     add 2eed4a54 feat(azblob): Add override_content_disposition support (#2065)
     add 55903bc8 feat(core): test for read_with_override_content_composition (#2067)
     add bf9dad6b feat(core): Add `start-after` support for list (#2071)
     add ca53000e refactor: Allow reusing the same operator to speed up tests (#2068)
     add fdc82f97 Bump to version 0.33 (#2074)
     add 85dcff72 chore(bindings/nodejs): update `package.json` repository info (#2078)
     add bd01753e chore: add tencent COS as a kind of s3 compatible backend in doc (#2081)
     add b5610e40 RFC-2083: Writer Sink API (#2083)
     add fa9e5d1b chore(bindings/nodejs): Add s3 example (#2082)
     add c541811c core: Add behavior test for read_with_if_match & stat_with_if_match (#2088)
     add 0e887f76 chore: improve LoggingLayer docs and pub use log::Level (#2089)
     add 2fed1a18 feat(tests): Add fuzz test for writer without content length (#2100)
     add fdc8d8d2 feat: add if_none_match support for obs (#2103)
     add d9eba0ad feat(services/oss): Add server side encryption support for oss (#2092)
     add 5c278a8b feat(core): update errorKind `PreconditionFailed` to `ConditionNotMatch` (#2104)
     add f76b33cb docs(refactor): Add more detailed description of operator, accessor, and builder (#2094)
     add de8a6366 feat(services/s3): Add `start-after` support for list (#2096)
     add 88915b93 fix(services/gcs): set `content length=0` for gcs initiate_resumable_upload (#2110)
     add d6ce0494 fix(bindings/nodejs): Fix index.d.ts not updated (#2117)
     add 8fc4f895 ci: Bring hdfs test back (#2114)
     add ed1d3624 feat: gcs support cache control (#2116)
     add 8e12fed0 Bump to version 0.33.1 (#2123)
     add a1aa1d3e feat(core): add test for `stat_with_if_none_match` (#2122)
     add 2554897a feat(services/gcs): Add start-after support for list (#2107)
     add 53768017 feat(services/azblob): Add supporting presign (#2120)
     add 42f5d595 ci: Mark job as skipped if owner is not apache (#2128)
     add 5edc4fd8 feat(services/gcs): Add supporting presign support (#2126)
     add 781dd1a8 feat(java): connect rust async/await with java future (#2112)
     add a814d9f6 docs: add hdfs classpath related troubleshoot (#2130)
     add a7ca89df fix(services/gcs): Fix invalid url for gcs presign (#2132)
     add 72c8c02c fix(clippy): suppress dead_code check (#2135)
     add d8179d75 chore: Make clippy happy (#2137)
     add f9cdce38 ci: Enable native-tls to test gcs presign for workaround (#2138)
     add d3be656a feat(core): Add `cache-control` to Metadata (#2136)
     add 2f5e31d2 fix(services/gcs): Remove HOST header to avoid gcs RESET connection (#2139)
     add 4f2fc2ad test(core): test for `write_with_cache_control` (#2131)
     add 3c3463bc test(core): test for `write_with_content_type` (#2140)
     add c6a37ad7 test(core): test for `read_with_if_none_match` (#2141)
     add dd290ab7 feat(services/supabase): Add read/write/stat support for supabase (#2119)
     add e871a6f0 Bump to version 0.33.2 (#2148)
     add 69dd4546 feat(services/onedrive): Add read and write support for OneDrive (#2129)
     add fb858334 test(core): test for `read_with_override_cache_control` (#2155)
     add 770b8fad feat(http_util): Implement multipart/form-data support (#2157)
     add dd4a482d feat(http_util): Implement multipart/mixed support (#2161)
     add ea8e938a docs(website): try to add opendal logo (#2159)
     add ddad905d RFC-2133: Introduce Append API (#2133)
     add a3935f14 doc: update vision to be more clear (#2164)
     add dc61d4a6 ci: Use microsoft rust devcontainer instead (#2165)
     add ec988b07 refactor(java): refactor java code for java binding (#2145)
     add 677f1f41 ci(devcontainer): Install development deps (#2167)
     add 4aa54745 refactor(layers/logging): parsing level str (#2160)
     add 99ecca73 chore: set workspace default members (#2168)
     add a1270b0c docs: Refactor `Contributing` and add `Developing` (#2169)
     add ea731c17 docs: Merge DEVELOPING into CONTRIBUTING (#2172)
     add f2bbfed4 docs: fix some grammar errors in the doc of Operator (#2173)
     add e387fe8c docs(nodejs): Add CONTRIBUTING docs (#2174)
     add c5ff0aec chore(deps): bump clap from 4.1.11 to 4.2.5 (#2183)
     add f365028f chore(deps): bump futures from 0.3.27 to 0.3.28 (#2181)
     add 3a711b3d chore(deps): bump assert_cmd from 2.0.10 to 2.0.11 (#2180)
     add ae6f6716 fix(service/s3): set retryable on batch (#2171)
     add f12b05c1 feat(services/sftp): Add read/write/stat support for sftp (#2186)
     add 95a9e410 docs: Add CONTRIBUTING for python (#2188)
     add bf264084 feat(services/gdrive): Add read & write & delete support for GoogleDrive (#2184)
     add 46822e6a chore: Refactor behavior test (#2189)
     add 91544102 feat(services/vercel): Add vercel remote cache support (#2193)
     add 3623b4e6 refactor: Move not initiated logic to utils instead (#2196)
     add 1ace92e1 ci: Setup vercel artifacts integration tests (#2197)
     add 564ad8de feat(tests): Enable supabase integration tests (#2190)
     add f269cf0b fix(services/supabase): Supabase ci fix (#2200)
     add 0a6fc13a ci: Remove not used odev tools (#2202)
     add 01227cfb refactor(services/memcached): Rewrite memecached connection entirely (#2204)
     add 6cdd0c2c ci: Add tools to generate NOTICE and all deps licenses (#2205)
     add 1ff1b7b3 ci: use Temurin JDK 11 to build the bindings-java (#2213)
     add c1693f4f feat(core): merge scan and list (#2214)
     add 041c7ac5 chore: update readme for more information that is more friendly to newcomers (#2217)
     add 402aae72 Bump to version 0.33.3 (#2212)
     new 5619baa3 Merge remote-tracking branch 'origin/main' into add-s3-api
     new 95a1af82 Update

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .devcontainer/devcontainer.json                    |  40 +-
 Cargo.toml => .devcontainer/post_create.sh         |  40 +-
 .env.example                                       |  14 +
 bindings/ruby/Gemfile => .github/licenserc.yaml    |  28 +-
 .github/workflows/bindings_java.yml                |   6 +-
 .github/workflows/bindings_nodejs.yml              |   8 +-
 .github/workflows/service_test_azdfs.yml           |   2 +-
 .github/workflows/service_test_gcs.yml             |   1 +
 .github/workflows/service_test_ghac.yml            |   1 +
 .github/workflows/service_test_hdfs.yml            |  90 ++-
 .github/workflows/service_test_http.yml            |   2 +
 .github/workflows/service_test_ipfs.yml            |   2 +
 .github/workflows/service_test_obs.yml             |   1 +
 .github/workflows/service_test_oss.yml             |   1 +
 .github/workflows/service_test_s3.yml              |   3 +
 .github/workflows/typos.yml                        |   2 +-
 CHANGELOG.md                                       | 156 +++++
 CONTRIBUTING.md                                    |  99 +++-
 Cargo.lock                                         | 547 +++++++++++++++---
 Cargo.toml                                         |   5 +-
 NOTICE                                             | 621 ++++++++++++++++++++
 README.md                                          | 119 ++--
 bin/oay/README.md                                  |   4 +
 bin/oay/src/services/s3/service.rs                 |  59 +-
 bindings/c/Makefile                                |   2 +-
 bindings/c/README.md                               |   2 +-
 bindings/c/build.rs                                |  16 -
 bindings/c/include/opendal.h                       | 127 +++--
 bindings/java/Cargo.toml                           |   5 +
 bindings/java/pom.xml                              |  21 +-
 bindings/java/src/lib.rs                           | 146 ++++-
 .../src/main/java/org/apache/opendal/Metadata.java |  28 +-
 .../opendal/{Metadata.java => OpenDALObject.java}  |  32 +-
 .../src/main/java/org/apache/opendal/Operator.java |  56 +-
 .../apache/opendal/exception/OpenDALErrorCode.java |  67 +++
 .../opendal/exception/OpenDALException.java}       |  15 +-
 .../java/org/apache/opendal/AsyncStepsTest.java    |  12 +-
 .../java/org/apache/opendal/ExceptionTest.java     |  53 ++
 .../test/java/org/apache/opendal/StepsTest.java    |  12 +-
 .../src/test/resources/features/binding.feature    |   1 +
 bindings/nodejs/.node-version                      |   1 +
 bindings/nodejs/CONTRIBUTING.md                    |  78 +++
 bindings/nodejs/Cargo.toml                         |   2 +-
 bindings/nodejs/README.md                          |  35 +-
 bindings/nodejs/examples/README.md                 |  42 ++
 .../examples/example_s3.js}                        |  48 +-
 bindings/nodejs/index.d.ts                         |  18 +-
 bindings/nodejs/npm/darwin-arm64/package.json      |   3 +-
 bindings/nodejs/npm/darwin-x64/package.json        |   3 +-
 bindings/nodejs/npm/linux-x64-gnu/package.json     |   3 +-
 bindings/nodejs/npm/win32-x64-msvc/package.json    |   3 +-
 bindings/nodejs/package.json                       |   2 +-
 bindings/python/CONTRIBUTING.md                    |  75 +++
 bindings/python/Cargo.toml                         |   2 +-
 bindings/python/examples/object.ipynb              |  46 ++
 bindings/ruby/Cargo.toml                           |   4 +
 bindings/{nodejs => ruby}/build.rs                 |   6 +-
 core/CONTRIBUTING.md                               |   4 -
 core/Cargo.toml                                    |  16 +-
 core/README.md                                     |   1 +
 core/src/docs/comparisons/vs_object_store.md       |   2 +-
 core/src/docs/concepts.rs                          |  33 +-
 core/src/docs/internals/accessor.rs                |  28 +-
 core/src/docs/rfcs/2083_writer_sink_api.md         | 106 ++++
 core/src/docs/rfcs/2133_append_api.md              |  88 +++
 core/src/docs/rfcs/mod.rs                          |   6 +
 core/src/docs/upgrade.md                           |  17 +
 core/src/layers/complete.rs                        |  89 ++-
 core/src/layers/error_context.rs                   |   2 +-
 core/src/layers/immutable_index.rs                 |   8 +-
 core/src/layers/logging.rs                         |  69 ++-
 core/src/layers/madsim.rs                          | 204 ++++---
 core/src/layers/mod.rs                             |   3 -
 core/src/layers/retry.rs                           |  37 +-
 core/src/lib.rs                                    |   6 +-
 core/src/raw/accessor.rs                           | 162 ++----
 core/src/raw/adapters/kv/api.rs                    |  16 +-
 core/src/raw/adapters/kv/backend.rs                |  24 +-
 core/src/raw/http_util/body.rs                     |  23 +-
 core/src/raw/http_util/client.rs                   |  27 +-
 core/src/raw/http_util/header.rs                   |  25 +
 core/src/raw/http_util/mod.rs                      |   6 +
 core/src/raw/http_util/multipart.rs                | 634 +++++++++++++++++++++
 core/src/raw/mod.rs                                |   2 -
 core/src/raw/oio/into_reader/by_range.rs           |   5 +-
 core/src/raw/oio/to_flat_pager.rs                  |   6 +-
 core/src/raw/oio/write.rs                          |   6 -
 core/src/services/azblob/backend.rs                |  91 ++-
 core/src/services/azblob/batch.rs                  | 150 +----
 core/src/services/azblob/core.rs                   | 126 ++--
 core/src/services/azblob/error.rs                  |   4 +-
 core/src/services/azblob/writer.rs                 |   1 +
 core/src/services/azdfs/backend.rs                 |  17 +-
 core/src/services/azdfs/error.rs                   |   2 +-
 core/src/services/dashmap/backend.rs               |   9 +-
 core/src/services/fs/backend.rs                    |  23 +-
 core/src/services/ftp/backend.rs                   |  63 +-
 core/src/services/gcs/backend.rs                   |  91 ++-
 core/src/services/gcs/core.rs                      | 173 +++++-
 core/src/services/gcs/error.rs                     |   4 +-
 core/src/services/gcs/pager.rs                     |  29 +-
 core/src/services/gdrive/backend.rs                | 252 ++++++++
 core/src/services/{ipmfs => gdrive}/builder.rs     |  99 ++--
 core/src/services/{webdav => gdrive}/error.rs      |   0
 core/src/services/{ipmfs => gdrive}/mod.rs         |   5 +-
 core/src/services/{webdav => gdrive}/writer.rs     |  19 +-
 core/src/services/ghac/backend.rs                  |  11 +-
 core/src/services/hdfs/backend.rs                  |  24 +-
 core/src/services/hdfs/writer.rs                   |  43 +-
 core/src/services/http/backend.rs                  |  32 +-
 core/src/services/http/error.rs                    |   4 +-
 core/src/services/ipfs/backend.rs                  |  11 +-
 core/src/services/ipmfs/backend.rs                 |  22 +-
 core/src/services/ipmfs/writer.rs                  |   5 +-
 core/src/services/memcached/MIT-ascii.txt          |  20 -
 core/src/services/memcached/ascii.rs               | 171 +++---
 core/src/services/memcached/backend.rs             |  62 +-
 core/src/services/memory/backend.rs                |  13 +-
 core/src/services/mod.rs                           |  26 +
 core/src/services/moka/backend.rs                  |   8 +-
 core/src/services/obs/backend.rs                   |  51 +-
 core/src/services/obs/core.rs                      |  16 +
 core/src/services/obs/error.rs                     |   4 +-
 core/src/services/obs/writer.rs                    |   1 +
 core/src/services/onedrive/backend.rs              | 212 +++++++
 core/src/services/{ipmfs => onedrive}/builder.rs   |  98 ++--
 core/src/services/{webdav => onedrive}/error.rs    |   0
 core/src/services/{ipmfs => onedrive}/mod.rs       |   5 +-
 core/src/services/{webdav => onedrive}/writer.rs   |  21 +-
 core/src/services/oss/backend.rs                   | 128 ++++-
 core/src/services/oss/core.rs                      |  80 ++-
 core/src/services/oss/error.rs                     |   4 +-
 core/src/services/oss/writer.rs                    |   3 +-
 core/src/services/redis/backend.rs                 |  12 +-
 core/src/services/rocksdb/backend.rs               |   7 +-
 core/src/services/s3/backend.rs                    |  79 ++-
 core/src/services/s3/core.rs                       |  29 +-
 core/src/services/s3/error.rs                      |  42 +-
 core/src/services/s3/pager.rs                      |  20 +-
 core/src/services/s3/writer.rs                     |   5 +-
 core/src/services/sftp/backend.rs                  | 509 +++++++++++++++++
 core/src/services/sftp/error.rs                    |  91 +++
 core/src/services/{hdfs => sftp}/mod.rs            |   5 +-
 core/src/services/sftp/pager.rs                    | 101 ++++
 core/src/services/sftp/utils.rs                    | 108 ++++
 core/src/services/{ipmfs => sftp}/writer.rs        |  42 +-
 core/src/services/sled/backend.rs                  |  14 +-
 core/src/services/supabase/backend.rs              | 323 +++++++++++
 core/src/services/supabase/core.rs                 | 270 +++++++++
 core/src/services/supabase/error.rs                |  73 +++
 core/src/services/{s3 => supabase}/mod.rs          |   4 +-
 core/src/services/{azblob => supabase}/writer.rs   |  52 +-
 core/src/services/vercel_artifacts/backend.rs      | 144 +++++
 .../{ipmfs => vercel_artifacts}/builder.rs         |  98 ++--
 .../services/{webdav => vercel_artifacts}/error.rs |   0
 .../services/{ipmfs => vercel_artifacts}/mod.rs    |   5 +-
 .../{webdav => vercel_artifacts}/writer.rs         |  26 +-
 core/src/services/wasabi/backend.rs                |  39 +-
 core/src/services/wasabi/error.rs                  |   2 +-
 core/src/services/webdav/backend.rs                |  26 +-
 core/src/services/webdav/pager.rs                  |  11 +-
 core/src/services/webhdfs/backend.rs               |  14 +-
 core/src/types/builder.rs                          |  20 +-
 core/src/types/capability.rs                       | 179 ++++++
 core/src/types/error.rs                            |  16 +-
 core/src/types/metadata.rs                         |  87 ++-
 core/src/types/mod.rs                              |   3 +
 core/src/types/operator/blocking_operator.rs       |   8 +-
 core/src/types/operator/metadata.rs                |  23 +-
 core/src/types/operator/operator.rs                |  81 ++-
 core/src/types/ops.rs                              |  45 +-
 core/src/types/reader.rs                           |   6 +-
 core/src/types/scheme.rs                           |  16 +
 core/src/types/writer.rs                           |  19 +-
 core/tests/behavior/blocking_copy.rs               |  40 +-
 core/tests/behavior/blocking_list.rs               |  59 +-
 core/tests/behavior/blocking_read.rs               |  38 +-
 core/tests/behavior/blocking_rename.rs             |  40 +-
 core/tests/behavior/blocking_write.rs              |  38 +-
 core/tests/behavior/copy.rs                        |  38 +-
 core/tests/behavior/list.rs                        | 124 ++--
 core/tests/behavior/list_only.rs                   |  34 +-
 core/tests/behavior/main.rs                        | 116 ++--
 core/tests/behavior/presign.rs                     |  34 +-
 core/tests/behavior/read_only.rs                   | 165 +++++-
 core/tests/behavior/rename.rs                      |  38 +-
 core/tests/behavior/utils.rs                       | 104 +++-
 core/tests/behavior/write.rs                       | 453 ++++++++++++++-
 odev                                               |  42 --
 rust-toolchain.toml                                |   2 +-
 rustfmt.toml                                       |   7 +-
 scripts/NOTICE.tpl                                 |  24 +
 scripts/README.md                                  |  10 +-
 scripts/check.sh                                   |  57 +-
 scripts/devtools/Dockerfile                        |  59 --
 .../__init__.py => scripts/generate_notice.sh      |   8 +-
 website/docs/vision.md                             |  38 +-
 website/docusaurus.config.js                       |   9 +-
 website/static/img/logo.svg                        |   1 +
 website/static/img/logo_dark.svg                   |   1 +
 200 files changed, 8632 insertions(+), 2306 deletions(-)
 copy Cargo.toml => .devcontainer/post_create.sh (55%)
 copy bindings/ruby/Gemfile => .github/licenserc.yaml (73%)
 copy bindings/java/src/main/java/org/apache/opendal/{Metadata.java => OpenDALObject.java} (64%)
 create mode 100644 bindings/java/src/main/java/org/apache/opendal/exception/OpenDALErrorCode.java
 copy bindings/{nodejs/index.js => java/src/main/java/org/apache/opendal/exception/OpenDALException.java} (69%)
 create mode 100644 bindings/java/src/test/java/org/apache/opendal/ExceptionTest.java
 create mode 120000 bindings/java/src/test/resources/features/binding.feature
 create mode 100644 bindings/nodejs/.node-version
 create mode 100644 bindings/nodejs/CONTRIBUTING.md
 create mode 100644 bindings/nodejs/examples/README.md
 copy bindings/{java/src/main/java/org/apache/opendal/Metadata.java => nodejs/examples/example_s3.js} (60%)
 create mode 100644 bindings/python/CONTRIBUTING.md
 create mode 100644 bindings/python/examples/object.ipynb
 copy bindings/{nodejs => ruby}/build.rs (88%)
 create mode 100644 core/src/docs/rfcs/2083_writer_sink_api.md
 create mode 100644 core/src/docs/rfcs/2133_append_api.md
 create mode 100644 core/src/raw/http_util/multipart.rs
 create mode 100644 core/src/services/gdrive/backend.rs
 copy core/src/services/{ipmfs => gdrive}/builder.rs (53%)
 copy core/src/services/{webdav => gdrive}/error.rs (100%)
 copy core/src/services/{ipmfs => gdrive}/mod.rs (94%)
 copy core/src/services/{webdav => gdrive}/writer.rs (80%)
 delete mode 100644 core/src/services/memcached/MIT-ascii.txt
 create mode 100644 core/src/services/onedrive/backend.rs
 copy core/src/services/{ipmfs => onedrive}/builder.rs (53%)
 copy core/src/services/{webdav => onedrive}/error.rs (100%)
 copy core/src/services/{ipmfs => onedrive}/mod.rs (94%)
 copy core/src/services/{webdav => onedrive}/writer.rs (75%)
 create mode 100644 core/src/services/sftp/backend.rs
 create mode 100644 core/src/services/sftp/error.rs
 copy core/src/services/{hdfs => sftp}/mod.rs (94%)
 create mode 100644 core/src/services/sftp/pager.rs
 create mode 100644 core/src/services/sftp/utils.rs
 copy core/src/services/{ipmfs => sftp}/writer.rs (60%)
 create mode 100644 core/src/services/supabase/backend.rs
 create mode 100644 core/src/services/supabase/core.rs
 create mode 100644 core/src/services/supabase/error.rs
 copy core/src/services/{s3 => supabase}/mod.rs (94%)
 copy core/src/services/{azblob => supabase}/writer.rs (63%)
 create mode 100644 core/src/services/vercel_artifacts/backend.rs
 copy core/src/services/{ipmfs => vercel_artifacts}/builder.rs (50%)
 copy core/src/services/{webdav => vercel_artifacts}/error.rs (100%)
 copy core/src/services/{ipmfs => vercel_artifacts}/mod.rs (93%)
 copy core/src/services/{webdav => vercel_artifacts}/writer.rs (75%)
 create mode 100644 core/src/types/capability.rs
 delete mode 100755 odev
 create mode 100644 scripts/NOTICE.tpl
 delete mode 100644 scripts/devtools/Dockerfile
 copy bindings/python/python/opendal/__init__.py => scripts/generate_notice.sh (77%)
 mode change 100644 => 100755
 create mode 100644 website/static/img/logo.svg
 create mode 100644 website/static/img/logo_dark.svg


[incubator-opendal] 01/02: Merge remote-tracking branch 'origin/main' into add-s3-api

Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch add-s3-api
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git

commit 5619baa3e9ca03cda8444b6340d812c83975abf2
Merge: 454ffdac 402aae72
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Sat May 6 23:35:56 2023 +0800

    Merge remote-tracking branch 'origin/main' into add-s3-api
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>

 .devcontainer/devcontainer.json                    |  40 +-
 .../c/Makefile => .devcontainer/post_create.sh     |  37 +-
 .env.example                                       |  14 +
 rust-toolchain.toml => .github/licenserc.yaml      |  18 +-
 .github/workflows/bindings_java.yml                |   6 +-
 .github/workflows/bindings_nodejs.yml              |   8 +-
 .github/workflows/service_test_azdfs.yml           |   2 +-
 .github/workflows/service_test_gcs.yml             |   1 +
 .github/workflows/service_test_ghac.yml            |   1 +
 .github/workflows/service_test_hdfs.yml            |  90 ++-
 .github/workflows/service_test_http.yml            |   2 +
 .github/workflows/service_test_ipfs.yml            |   2 +
 .github/workflows/service_test_obs.yml             |   1 +
 .github/workflows/service_test_oss.yml             |   1 +
 .github/workflows/service_test_s3.yml              |   3 +
 .github/workflows/typos.yml                        |   2 +-
 CHANGELOG.md                                       | 156 +++++
 CONTRIBUTING.md                                    |  99 +++-
 Cargo.lock                                         | 547 +++++++++++++++---
 Cargo.toml                                         |   5 +-
 NOTICE                                             | 621 ++++++++++++++++++++
 README.md                                          | 119 ++--
 bindings/c/Makefile                                |   2 +-
 bindings/c/README.md                               |   2 +-
 bindings/c/build.rs                                |  16 -
 bindings/c/include/opendal.h                       | 127 +++--
 bindings/java/Cargo.toml                           |   5 +
 bindings/java/pom.xml                              |  21 +-
 bindings/java/src/lib.rs                           | 146 ++++-
 .../src/main/java/org/apache/opendal/Metadata.java |  28 +-
 .../opendal/{Metadata.java => OpenDALObject.java}  |  32 +-
 .../src/main/java/org/apache/opendal/Operator.java |  56 +-
 .../apache/opendal/exception/OpenDALErrorCode.java |  67 +++
 .../OpenDALException.java}                         |  32 +-
 .../java/org/apache/opendal/AsyncStepsTest.java    |  12 +-
 .../java/org/apache/opendal/ExceptionTest.java     |  53 ++
 .../test/java/org/apache/opendal/StepsTest.java    |  12 +-
 .../src/test/resources/features/binding.feature    |   1 +
 bindings/nodejs/.node-version                      |   1 +
 bindings/nodejs/CONTRIBUTING.md                    |  78 +++
 bindings/nodejs/Cargo.toml                         |   2 +-
 bindings/nodejs/README.md                          |  35 +-
 bindings/nodejs/examples/README.md                 |  42 ++
 .../examples/example_s3.js}                        |  48 +-
 bindings/nodejs/index.d.ts                         |  18 +-
 bindings/nodejs/npm/darwin-arm64/package.json      |   3 +-
 bindings/nodejs/npm/darwin-x64/package.json        |   3 +-
 bindings/nodejs/npm/linux-x64-gnu/package.json     |   3 +-
 bindings/nodejs/npm/win32-x64-msvc/package.json    |   3 +-
 bindings/nodejs/package.json                       |   2 +-
 bindings/python/CONTRIBUTING.md                    |  75 +++
 bindings/python/Cargo.toml                         |   2 +-
 bindings/python/examples/object.ipynb              |  46 ++
 bindings/ruby/Cargo.toml                           |   4 +
 core/src/types/mod.rs => bindings/ruby/build.rs    |  43 +-
 core/CONTRIBUTING.md                               |   4 -
 core/Cargo.toml                                    |  16 +-
 core/README.md                                     |   1 +
 core/src/docs/comparisons/vs_object_store.md       |   2 +-
 core/src/docs/concepts.rs                          |  33 +-
 core/src/docs/internals/accessor.rs                |  28 +-
 core/src/docs/rfcs/2083_writer_sink_api.md         | 106 ++++
 core/src/docs/rfcs/2133_append_api.md              |  88 +++
 core/src/docs/rfcs/mod.rs                          |   6 +
 core/src/docs/upgrade.md                           |  17 +
 core/src/layers/complete.rs                        |  89 ++-
 core/src/layers/error_context.rs                   |   2 +-
 core/src/layers/immutable_index.rs                 |   8 +-
 core/src/layers/logging.rs                         |  69 ++-
 core/src/layers/madsim.rs                          | 204 ++++---
 core/src/layers/mod.rs                             |   3 -
 core/src/layers/retry.rs                           |  37 +-
 core/src/lib.rs                                    |   6 +-
 core/src/raw/accessor.rs                           | 162 ++----
 core/src/raw/adapters/kv/api.rs                    |  16 +-
 core/src/raw/adapters/kv/backend.rs                |  24 +-
 core/src/raw/http_util/body.rs                     |  23 +-
 core/src/raw/http_util/client.rs                   |  27 +-
 core/src/raw/http_util/header.rs                   |  25 +
 core/src/raw/http_util/mod.rs                      |   6 +
 core/src/raw/http_util/multipart.rs                | 634 +++++++++++++++++++++
 core/src/raw/mod.rs                                |   2 -
 core/src/raw/oio/into_reader/by_range.rs           |   5 +-
 core/src/raw/oio/to_flat_pager.rs                  |   6 +-
 core/src/raw/oio/write.rs                          |   6 -
 core/src/services/azblob/backend.rs                |  91 ++-
 core/src/services/azblob/batch.rs                  | 150 +----
 core/src/services/azblob/core.rs                   | 126 ++--
 core/src/services/azblob/error.rs                  |   4 +-
 core/src/services/azblob/writer.rs                 |   1 +
 core/src/services/azdfs/backend.rs                 |  17 +-
 core/src/services/azdfs/error.rs                   |   2 +-
 core/src/services/dashmap/backend.rs               |   9 +-
 core/src/services/fs/backend.rs                    |  23 +-
 core/src/services/ftp/backend.rs                   |  63 +-
 core/src/services/gcs/backend.rs                   |  91 ++-
 core/src/services/gcs/core.rs                      | 173 +++++-
 core/src/services/gcs/error.rs                     |   4 +-
 core/src/services/gcs/pager.rs                     |  29 +-
 core/src/services/gdrive/backend.rs                | 252 ++++++++
 core/src/services/gdrive/builder.rs                | 148 +++++
 core/src/services/{http => gdrive}/error.rs        |   1 -
 core/src/{types => services/gdrive}/mod.rs         |  40 +-
 core/src/services/{ipmfs => gdrive}/writer.rs      |  25 +-
 core/src/services/ghac/backend.rs                  |  11 +-
 core/src/services/hdfs/backend.rs                  |  24 +-
 core/src/services/hdfs/writer.rs                   |  43 +-
 core/src/services/http/backend.rs                  |  32 +-
 core/src/services/http/error.rs                    |   4 +-
 core/src/services/ipfs/backend.rs                  |  11 +-
 core/src/services/ipmfs/backend.rs                 |  22 +-
 core/src/services/ipmfs/writer.rs                  |   5 +-
 core/src/services/memcached/MIT-ascii.txt          |  20 -
 core/src/services/memcached/ascii.rs               | 171 +++---
 core/src/services/memcached/backend.rs             |  62 +-
 core/src/services/memory/backend.rs                |  13 +-
 core/src/services/mod.rs                           |  26 +
 core/src/services/moka/backend.rs                  |   8 +-
 core/src/services/obs/backend.rs                   |  51 +-
 core/src/services/obs/core.rs                      |  16 +
 core/src/services/obs/error.rs                     |   4 +-
 core/src/services/obs/writer.rs                    |   1 +
 core/src/services/onedrive/backend.rs              | 212 +++++++
 core/src/services/onedrive/builder.rs              | 149 +++++
 core/src/services/{http => onedrive}/error.rs      |   1 -
 core/src/{types => services/onedrive}/mod.rs       |  40 +-
 core/src/services/{ipmfs => onedrive}/writer.rs    |  27 +-
 core/src/services/oss/backend.rs                   | 128 ++++-
 core/src/services/oss/core.rs                      |  80 ++-
 core/src/services/oss/error.rs                     |   4 +-
 core/src/services/oss/writer.rs                    |   3 +-
 core/src/services/redis/backend.rs                 |  12 +-
 core/src/services/rocksdb/backend.rs               |   7 +-
 core/src/services/s3/backend.rs                    |  79 ++-
 core/src/services/s3/core.rs                       |  29 +-
 core/src/services/s3/error.rs                      |  42 +-
 core/src/services/s3/pager.rs                      |  20 +-
 core/src/services/s3/writer.rs                     |   5 +-
 core/src/services/sftp/backend.rs                  | 509 +++++++++++++++++
 core/src/services/sftp/error.rs                    |  91 +++
 core/src/{types => services/sftp}/mod.rs           |  43 +-
 core/src/services/sftp/pager.rs                    | 101 ++++
 core/src/services/sftp/utils.rs                    | 108 ++++
 core/src/services/{ipmfs => sftp}/writer.rs        |  42 +-
 core/src/services/sled/backend.rs                  |  14 +-
 core/src/services/supabase/backend.rs              | 323 +++++++++++
 core/src/services/supabase/core.rs                 | 270 +++++++++
 core/src/services/supabase/error.rs                |  73 +++
 core/src/{types => services/supabase}/mod.rs       |  43 +-
 core/src/services/{azblob => supabase}/writer.rs   |  52 +-
 core/src/services/vercel_artifacts/backend.rs      | 144 +++++
 core/src/services/vercel_artifacts/builder.rs      | 123 ++++
 .../services/{http => vercel_artifacts}/error.rs   |   1 -
 .../{types => services/vercel_artifacts}/mod.rs    |  40 +-
 .../services/{ipmfs => vercel_artifacts}/writer.rs |  24 +-
 core/src/services/wasabi/backend.rs                |  39 +-
 core/src/services/wasabi/error.rs                  |   2 +-
 core/src/services/webdav/backend.rs                |  26 +-
 core/src/services/webdav/pager.rs                  |  11 +-
 core/src/services/webhdfs/backend.rs               |  14 +-
 core/src/types/builder.rs                          |  20 +-
 core/src/types/capability.rs                       | 179 ++++++
 core/src/types/error.rs                            |  16 +-
 core/src/types/metadata.rs                         |  87 ++-
 core/src/types/mod.rs                              |   3 +
 core/src/types/operator/blocking_operator.rs       |   8 +-
 core/src/types/operator/metadata.rs                |  23 +-
 core/src/types/operator/operator.rs                |  81 ++-
 core/src/types/ops.rs                              |  45 +-
 core/src/types/reader.rs                           |   6 +-
 core/src/types/scheme.rs                           |  16 +
 core/src/types/writer.rs                           |  19 +-
 core/tests/behavior/blocking_copy.rs               |  40 +-
 core/tests/behavior/blocking_list.rs               |  59 +-
 core/tests/behavior/blocking_read.rs               |  38 +-
 core/tests/behavior/blocking_rename.rs             |  40 +-
 core/tests/behavior/blocking_write.rs              |  38 +-
 core/tests/behavior/copy.rs                        |  38 +-
 core/tests/behavior/list.rs                        | 124 ++--
 core/tests/behavior/list_only.rs                   |  34 +-
 core/tests/behavior/main.rs                        | 116 ++--
 core/tests/behavior/presign.rs                     |  34 +-
 core/tests/behavior/read_only.rs                   | 165 +++++-
 core/tests/behavior/rename.rs                      |  38 +-
 core/tests/behavior/utils.rs                       | 104 +++-
 core/tests/behavior/write.rs                       | 453 ++++++++++++++-
 odev                                               |  42 --
 rust-toolchain.toml                                |   2 +-
 rustfmt.toml                                       |   7 +-
 scripts/NOTICE.tpl                                 |  24 +
 scripts/README.md                                  |  10 +-
 scripts/check.sh                                   |  57 +-
 scripts/devtools/Dockerfile                        |  59 --
 rust-toolchain.toml => scripts/generate_notice.sh  |  10 +-
 website/docs/vision.md                             |  38 +-
 website/docusaurus.config.js                       |   9 +-
 website/static/img/logo.svg                        |   1 +
 website/static/img/logo_dark.svg                   |   1 +
 198 files changed, 8921 insertions(+), 2317 deletions(-)

diff --cc Cargo.lock
index 3a54fff4,9ed71f03..29253a12
--- a/Cargo.lock
+++ b/Cargo.lock
@@@ -263,55 -318,22 +318,71 @@@ version = "1.1.0
  source = "registry+https://github.com/rust-lang/crates.io-index"
  checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
  
+ [[package]]
+ name = "awaitable"
+ version = "0.4.0"
+ source = "registry+https://github.com/rust-lang/crates.io-index"
+ checksum = "70af449c9a763cb655c6a1e5338b42d99c67190824ff90658c1e30be844c0775"
+ dependencies = [
+  "awaitable-error",
+  "cfg-if",
+ ]
+ 
+ [[package]]
+ name = "awaitable-error"
+ version = "0.1.0"
+ source = "registry+https://github.com/rust-lang/crates.io-index"
+ checksum = "d5b3469636cdf8543cceab175efca534471f36eee12fb8374aba00eb5e7e7f8a"
+ 
 +[[package]]
 +name = "axum"
- version = "0.6.16"
++version = "0.6.18"
 +source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "113713495a32dd0ab52baf5c10044725aa3aec00b31beda84218e469029b72a3"
++checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39"
 +dependencies = [
 + "async-trait",
 + "axum-core",
 + "bitflags 1.3.2",
 + "bytes",
 + "futures-util",
 + "http",
 + "http-body",
 + "hyper",
 + "itoa",
 + "matchit",
 + "memchr",
 + "mime",
 + "percent-encoding",
 + "pin-project-lite",
 + "rustversion",
 + "serde",
 + "serde_json",
 + "serde_path_to_error",
 + "serde_urlencoded",
 + "sync_wrapper",
 + "tokio",
 + "tower",
 + "tower-layer",
 + "tower-service",
 +]
 +
 +[[package]]
 +name = "axum-core"
 +version = "0.3.4"
 +source = "registry+https://github.com/rust-lang/crates.io-index"
 +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
 +dependencies = [
 + "async-trait",
 + "bytes",
 + "futures-util",
 + "http",
 + "http-body",
 + "mime",
 + "rustversion",
 + "tower-layer",
 + "tower-service",
 +]
 +
  [[package]]
  name = "backon"
  version = "0.4.0"
@@@ -2354,16 -2427,15 +2488,16 @@@ dependencies = 
  
  [[package]]
  name = "oay"
- version = "0.32.0"
+ version = "0.33.3"
  dependencies = [
   "anyhow",
 + "axum",
 + "chrono",
-  "clap 4.1.11",
-  "dirs",
+  "clap 4.2.5",
+  "dirs 5.0.0",
 - "env_logger",
   "futures",
 - "log",
   "opendal",
 + "quick-xml 0.27.1",
   "serde",
   "tokio",
   "toml 0.7.3",


[incubator-opendal] 02/02: Update

Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch add-s3-api
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git

commit 95a1af8296b8e29fdd93fc3cb055ab1c07f8b4f4
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Sat May 6 23:42:57 2023 +0800

    Update
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 bin/oay/README.md                  |  4 +++
 bin/oay/src/services/s3/service.rs | 59 ++++++++------------------------------
 2 files changed, 16 insertions(+), 47 deletions(-)

diff --git a/bin/oay/README.md b/bin/oay/README.md
index caa23e57..766fd958 100644
--- a/bin/oay/README.md
+++ b/bin/oay/README.md
@@ -11,3 +11,7 @@ Allow users to access different storage backend through their preferred APIs.
 ## Status
 
 Our first milestone is to provide S3 APIs.
+
+### S3 API
+
+Only `list_object_v2` with `start_after` is supported.
diff --git a/bin/oay/src/services/s3/service.rs b/bin/oay/src/services/s3/service.rs
index efb546d8..e1c76259 100644
--- a/bin/oay/src/services/s3/service.rs
+++ b/bin/oay/src/services/s3/service.rs
@@ -15,11 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::collections::HashMap;
 use std::sync::Arc;
-use std::sync::Mutex;
 
-use anyhow::anyhow;
 use axum::extract::Query;
 use axum::extract::State;
 use axum::http::StatusCode;
@@ -28,7 +25,7 @@ use axum::response::Response;
 use axum::routing::get;
 use axum::Router;
 use chrono::SecondsFormat;
-use opendal::Lister;
+use opendal::ops::OpList;
 use opendal::Metakey;
 use opendal::Operator;
 use serde::Deserialize;
@@ -36,7 +33,6 @@ use serde::Serialize;
 use tower::ServiceBuilder;
 use tower_http::trace::TraceLayer;
 use tracing::debug;
-use uuid::Uuid;
 
 use crate::Config;
 
@@ -58,7 +54,6 @@ impl S3Service {
             .layer(ServiceBuilder::new().layer(TraceLayer::new_for_http()))
             .with_state(S3State {
                 op: self.op.clone(),
-                list_objects: Arc::default(),
             });
 
         axum::Server::bind(&s3_cfg.addr.parse().unwrap())
@@ -72,8 +67,6 @@ impl S3Service {
 #[derive(Clone)]
 pub struct S3State {
     op: Operator,
-    /// TODO: remove this global lock by page checkpoint.
-    list_objects: Arc<Mutex<HashMap<String, Lister>>>,
 }
 
 /// # TODO
@@ -81,13 +74,12 @@ pub struct S3State {
 /// we need to support following parameters:
 ///
 /// - max-keys
-/// - start-after
+/// - continuation_token
 #[derive(Deserialize, Default, Debug)]
 #[serde(default)]
 struct ListObjectsV2Params {
     prefix: String,
-    delimiter: String,
-    continuation_token: String,
+    start_after: String,
 }
 
 async fn handle_list_objects(
@@ -95,26 +87,14 @@ async fn handle_list_objects(
     params: Query<ListObjectsV2Params>,
 ) -> Result<OkResponse, ErrorResponse> {
     debug!("got params: {:?}", params);
-    if params.delimiter != "/" && !params.delimiter.is_empty() {
-        return Err(anyhow!("delimiter is not supported").into());
-    }
-
-    let lister = state
-        .list_objects
-        .lock()
-        .unwrap()
-        .remove(format!("{}-{}", params.prefix, params.continuation_token).as_str());
 
-    let mut lister = match lister {
-        Some(lister) => lister,
-        None => {
-            if params.delimiter.is_empty() {
-                state.op.list(&params.prefix).await?
-            } else {
-                state.op.scan(&params.prefix).await?
-            }
-        }
-    };
+    let mut lister = state
+        .op
+        .list_with(
+            &params.prefix,
+            OpList::new().with_start_after(&params.start_after),
+        )
+        .await?;
 
     let page = lister.next_page().await?.unwrap_or_default();
 
@@ -147,25 +127,11 @@ async fn handle_list_objects(
         }
     }
 
-    let next_continuation_token = if is_truncated {
-        let token = format!("{}-{}", params.prefix, Uuid::new_v4().as_u128());
-        // Insert the lister into the state so that we can continue listing
-        state
-            .list_objects
-            .lock()
-            .unwrap()
-            .insert(token.clone(), lister);
-        token
-    } else {
-        String::new()
-    };
-
     let resp = ListBucketResult {
         is_truncated,
         common_prefixes,
         contents,
-        continuation_token: params.continuation_token.to_string(),
-        next_continuation_token,
+        start_after: Some(params.start_after.clone()),
     };
 
     Ok(OkResponse {
@@ -180,8 +146,7 @@ struct ListBucketResult {
     is_truncated: bool,
     common_prefixes: Vec<CommonPrefix>,
     contents: Vec<Object>,
-    continuation_token: String,
-    next_continuation_token: String,
+    start_after: Option<String>,
 }
 
 #[derive(Serialize, Default)]