You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/12/25 03:53:15 UTC

(incubator-opendal) branch fix-gcs created (now c26eecb2f)

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a change to branch fix-gcs
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git


      at c26eecb2f Add CI

This branch includes the following new commits:

     new 407712595 Fix build on gcs
     new ee1cd4f63 Merge remote-tracking branch 'origin/main' into fix-gcs
     new c26eecb2f Add CI

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(incubator-opendal) 03/03: Add CI

Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch fix-gcs
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git

commit c26eecb2f0e3a0239094968495b804d752355252
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Dec 25 11:53:06 2023 +0800

    Add CI
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 .github/workflows/ci.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6bd9a3b84..cdf452be7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -293,7 +293,7 @@ jobs:
           )
           cargo build --features "${FEATURES[*]}"
 
-  # We only support some services(see `available_services` below) for now, but we will extend wasm support for other services.
+  # We only support some services(see `available_services` below) for now.
   build_under_wasm:
     runs-on: ubuntu-latest
     steps:
@@ -304,14 +304,13 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Build
         run: |
-          available_services=(
+          FEATURES=(
             services-azblob
             services-gdrive
             services-s3
           )
-          IFS=","
           rustup target add wasm32-unknown-unknown
-          cargo build --target wasm32-unknown-unknown --no-default-features --features="${available_services[*]}"
+          cargo build --target wasm32-unknown-unknown --no-default-features --features="${FEATURES[*]}"
 
   unit:
     runs-on: ubuntu-latest


(incubator-opendal) 02/03: Merge remote-tracking branch 'origin/main' into fix-gcs

Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch fix-gcs
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git

commit ee1cd4f63d8f12dfd46bd999c6c22a3f811cff72
Merge: 407712595 4c64a8096
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Dec 25 11:51:42 2023 +0800

    Merge remote-tracking branch 'origin/main' into fix-gcs

 .github/workflows/ci.yml              |  3 +-
 .github/workflows/edge_test.yml       | 45 ++++++++++++++++
 bindings/ocaml/Cargo.lock             | 16 +++---
 bindings/ocaml/Cargo.toml             |  2 +-
 bindings/ocaml/README.md              |  8 +--
 core/edge/s3_read_on_wasm/src/lib.rs  |  4 +-
 core/src/raw/http_util/header.rs      | 16 ++++++
 core/src/raw/http_util/mod.rs         |  1 +
 core/src/services/dbfs/docs.md        |  2 +-
 core/src/services/gdrive/backend.rs   |  3 +-
 core/src/services/gdrive/lister.rs    |  3 +-
 core/src/services/gdrive/writer.rs    |  3 +-
 core/src/services/huggingface/docs.md |  4 +-
 core/src/services/swift/docs.md       |  2 +-
 core/src/services/upyun/core.rs       | 96 ++++-------------------------------
 core/src/services/upyun/writer.rs     | 25 +++++----
 website/docs/services/huggingface.mdx | 76 +++++++++++++++++++++++++++
 17 files changed, 188 insertions(+), 121 deletions(-)


(incubator-opendal) 01/03: Fix build on gcs

Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch fix-gcs
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git

commit 40771259515fbf204eb36011c461e7319efc01eb
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Dec 25 11:51:32 2023 +0800

    Fix build on gcs
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 core/Cargo.toml                       |  2 +-
 core/src/raw/oio/write/range_write.rs | 17 +++++++++++------
 core/src/services/gcs/backend.rs      | 10 ++++++++--
 core/src/services/gcs/lister.rs       |  3 ++-
 core/src/services/gcs/writer.rs       |  3 ++-
 5 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/core/Cargo.toml b/core/Cargo.toml
index 3dd0438ea..111573c41 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -329,4 +329,4 @@ tracing-subscriber = { version = "0.3", features = [
   "env-filter",
   "tracing-log",
 ] }
-wiremock = "0.5"
\ No newline at end of file
+wiremock = "0.5"
diff --git a/core/src/raw/oio/write/range_write.rs b/core/src/raw/oio/write/range_write.rs
index 3872d1cd5..8e6c72f61 100644
--- a/core/src/raw/oio/write/range_write.rs
+++ b/core/src/raw/oio/write/range_write.rs
@@ -20,7 +20,6 @@ use std::task::Context;
 use std::task::Poll;
 
 use async_trait::async_trait;
-use futures::future::BoxFuture;
 use futures::FutureExt;
 
 use crate::raw::oio::WriteBuf;
@@ -48,7 +47,8 @@ use crate::*;
 /// - Services impl `RangeWrite`
 /// - `RangeWriter` impl `Write`
 /// - Expose `RangeWriter` as `Accessor::Writer`
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 pub trait RangeWrite: Send + Sync + Unpin + 'static {
     /// write_once is used to write the data to underlying storage at once.
     ///
@@ -93,12 +93,17 @@ pub struct RangeWriter<W: RangeWrite> {
 
 enum State<W> {
     Idle(Option<W>),
-    Init(BoxFuture<'static, (W, Result<String>)>),
-    Write(BoxFuture<'static, (W, Result<u64>)>),
-    Complete(BoxFuture<'static, (W, Result<()>)>),
-    Abort(BoxFuture<'static, (W, Result<()>)>),
+    Init(BoxedFuture<(W, Result<String>)>),
+    Write(BoxedFuture<(W, Result<u64>)>),
+    Complete(BoxedFuture<(W, Result<()>)>),
+    Abort(BoxedFuture<(W, Result<()>)>),
 }
 
+/// # Safety
+///
+/// wasm32 is a special target that we only have one event-loop for this state.
+unsafe impl<S: RangeWrite> Send for State<S> {}
+
 /// # Safety
 ///
 /// We will only take `&mut Self` reference for State.
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index 60c8c6b2f..6a0160123 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -34,8 +34,8 @@ use super::core::*;
 use super::error::parse_error;
 use super::lister::GcsLister;
 use super::writer::GcsWriter;
+use super::writer::GcsWriters;
 use crate::raw::*;
-use crate::services::gcs::writer::GcsWriters;
 use crate::*;
 
 const DEFAULT_GCS_ENDPOINT: &str = "https://storage.googleapis.com";
@@ -269,6 +269,11 @@ impl Builder for GcsBuilder {
         if let Some(cred) = &self.config.credential_path {
             cred_loader = cred_loader.with_path(cred);
         }
+        #[cfg(target_arch = "wasm32")]
+        {
+            cred_loader = cred_loader.with_disable_env();
+            cred_loader = cred_loader.with_disable_well_known_location();
+        }
 
         let scope = if let Some(scope) = &self.config.scope {
             scope
@@ -313,7 +318,8 @@ pub struct GcsBackend {
     core: Arc<GcsCore>,
 }
 
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 impl Accessor for GcsBackend {
     type Reader = IncomingAsyncBody;
     type BlockingReader = ();
diff --git a/core/src/services/gcs/lister.rs b/core/src/services/gcs/lister.rs
index 218518716..c7da1a86e 100644
--- a/core/src/services/gcs/lister.rs
+++ b/core/src/services/gcs/lister.rs
@@ -60,7 +60,8 @@ impl GcsLister {
     }
 }
 
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 impl oio::PageList for GcsLister {
     async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
         let resp = self
diff --git a/core/src/services/gcs/writer.rs b/core/src/services/gcs/writer.rs
index d55d6d7a9..99b6217f2 100644
--- a/core/src/services/gcs/writer.rs
+++ b/core/src/services/gcs/writer.rs
@@ -43,7 +43,8 @@ impl GcsWriter {
     }
 }
 
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 impl oio::RangeWrite for GcsWriter {
     async fn write_once(&self, size: u64, body: AsyncBody) -> Result<()> {
         let mut req = self.core.gcs_insert_object_request(