You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/12/25 03:53:15 UTC
(incubator-opendal) branch fix-gcs created (now c26eecb2f)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a change to branch fix-gcs
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
at c26eecb2f Add CI
This branch includes the following new commits:
new 407712595 Fix build on gcs
new ee1cd4f63 Merge remote-tracking branch 'origin/main' into fix-gcs
new c26eecb2f Add CI
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-opendal) 03/03: Add CI
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch fix-gcs
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit c26eecb2f0e3a0239094968495b804d752355252
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Dec 25 11:53:06 2023 +0800
Add CI
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
.github/workflows/ci.yml | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6bd9a3b84..cdf452be7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -293,7 +293,7 @@ jobs:
)
cargo build --features "${FEATURES[*]}"
- # We only support some services(see `available_services` below) for now, but we will extend wasm support for other services.
+ # We only support some services(see `available_services` below) for now.
build_under_wasm:
runs-on: ubuntu-latest
steps:
@@ -304,14 +304,13 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Build
run: |
- available_services=(
+ FEATURES=(
services-azblob
services-gdrive
services-s3
)
- IFS=","
rustup target add wasm32-unknown-unknown
- cargo build --target wasm32-unknown-unknown --no-default-features --features="${available_services[*]}"
+ cargo build --target wasm32-unknown-unknown --no-default-features --features="${FEATURES[*]}"
unit:
runs-on: ubuntu-latest
(incubator-opendal) 02/03: Merge remote-tracking branch 'origin/main' into fix-gcs
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch fix-gcs
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit ee1cd4f63d8f12dfd46bd999c6c22a3f811cff72
Merge: 407712595 4c64a8096
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Dec 25 11:51:42 2023 +0800
Merge remote-tracking branch 'origin/main' into fix-gcs
.github/workflows/ci.yml | 3 +-
.github/workflows/edge_test.yml | 45 ++++++++++++++++
bindings/ocaml/Cargo.lock | 16 +++---
bindings/ocaml/Cargo.toml | 2 +-
bindings/ocaml/README.md | 8 +--
core/edge/s3_read_on_wasm/src/lib.rs | 4 +-
core/src/raw/http_util/header.rs | 16 ++++++
core/src/raw/http_util/mod.rs | 1 +
core/src/services/dbfs/docs.md | 2 +-
core/src/services/gdrive/backend.rs | 3 +-
core/src/services/gdrive/lister.rs | 3 +-
core/src/services/gdrive/writer.rs | 3 +-
core/src/services/huggingface/docs.md | 4 +-
core/src/services/swift/docs.md | 2 +-
core/src/services/upyun/core.rs | 96 ++++-------------------------------
core/src/services/upyun/writer.rs | 25 +++++----
website/docs/services/huggingface.mdx | 76 +++++++++++++++++++++++++++
17 files changed, 188 insertions(+), 121 deletions(-)
(incubator-opendal) 01/03: Fix build on gcs
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch fix-gcs
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 40771259515fbf204eb36011c461e7319efc01eb
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Dec 25 11:51:32 2023 +0800
Fix build on gcs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/Cargo.toml | 2 +-
core/src/raw/oio/write/range_write.rs | 17 +++++++++++------
core/src/services/gcs/backend.rs | 10 ++++++++--
core/src/services/gcs/lister.rs | 3 ++-
core/src/services/gcs/writer.rs | 3 ++-
5 files changed, 24 insertions(+), 11 deletions(-)
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 3dd0438ea..111573c41 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -329,4 +329,4 @@ tracing-subscriber = { version = "0.3", features = [
"env-filter",
"tracing-log",
] }
-wiremock = "0.5"
\ No newline at end of file
+wiremock = "0.5"
diff --git a/core/src/raw/oio/write/range_write.rs b/core/src/raw/oio/write/range_write.rs
index 3872d1cd5..8e6c72f61 100644
--- a/core/src/raw/oio/write/range_write.rs
+++ b/core/src/raw/oio/write/range_write.rs
@@ -20,7 +20,6 @@ use std::task::Context;
use std::task::Poll;
use async_trait::async_trait;
-use futures::future::BoxFuture;
use futures::FutureExt;
use crate::raw::oio::WriteBuf;
@@ -48,7 +47,8 @@ use crate::*;
/// - Services impl `RangeWrite`
/// - `RangeWriter` impl `Write`
/// - Expose `RangeWriter` as `Accessor::Writer`
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
pub trait RangeWrite: Send + Sync + Unpin + 'static {
/// write_once is used to write the data to underlying storage at once.
///
@@ -93,12 +93,17 @@ pub struct RangeWriter<W: RangeWrite> {
enum State<W> {
Idle(Option<W>),
- Init(BoxFuture<'static, (W, Result<String>)>),
- Write(BoxFuture<'static, (W, Result<u64>)>),
- Complete(BoxFuture<'static, (W, Result<()>)>),
- Abort(BoxFuture<'static, (W, Result<()>)>),
+ Init(BoxedFuture<(W, Result<String>)>),
+ Write(BoxedFuture<(W, Result<u64>)>),
+ Complete(BoxedFuture<(W, Result<()>)>),
+ Abort(BoxedFuture<(W, Result<()>)>),
}
+/// # Safety
+///
+/// wasm32 is a special target that we only have one event-loop for this state.
+unsafe impl<S: RangeWrite> Send for State<S> {}
+
/// # Safety
///
/// We will only take `&mut Self` reference for State.
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index 60c8c6b2f..6a0160123 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -34,8 +34,8 @@ use super::core::*;
use super::error::parse_error;
use super::lister::GcsLister;
use super::writer::GcsWriter;
+use super::writer::GcsWriters;
use crate::raw::*;
-use crate::services::gcs::writer::GcsWriters;
use crate::*;
const DEFAULT_GCS_ENDPOINT: &str = "https://storage.googleapis.com";
@@ -269,6 +269,11 @@ impl Builder for GcsBuilder {
if let Some(cred) = &self.config.credential_path {
cred_loader = cred_loader.with_path(cred);
}
+ #[cfg(target_arch = "wasm32")]
+ {
+ cred_loader = cred_loader.with_disable_env();
+ cred_loader = cred_loader.with_disable_well_known_location();
+ }
let scope = if let Some(scope) = &self.config.scope {
scope
@@ -313,7 +318,8 @@ pub struct GcsBackend {
core: Arc<GcsCore>,
}
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
impl Accessor for GcsBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
diff --git a/core/src/services/gcs/lister.rs b/core/src/services/gcs/lister.rs
index 218518716..c7da1a86e 100644
--- a/core/src/services/gcs/lister.rs
+++ b/core/src/services/gcs/lister.rs
@@ -60,7 +60,8 @@ impl GcsLister {
}
}
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
impl oio::PageList for GcsLister {
async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
let resp = self
diff --git a/core/src/services/gcs/writer.rs b/core/src/services/gcs/writer.rs
index d55d6d7a9..99b6217f2 100644
--- a/core/src/services/gcs/writer.rs
+++ b/core/src/services/gcs/writer.rs
@@ -43,7 +43,8 @@ impl GcsWriter {
}
}
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
impl oio::RangeWrite for GcsWriter {
async fn write_once(&self, size: u64, body: AsyncBody) -> Result<()> {
let mut req = self.core.gcs_insert_object_request(