You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by "hoslo (via GitHub)" <gi...@apache.org> on 2023/12/27 09:52:13 UTC
[PR] feat(services): add chainsafe support [incubator-opendal]
hoslo opened a new pull request, #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834
Part #3823
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436992095
##########
core/src/types/scheme.rs:
##########
@@ -44,6 +44,8 @@ pub enum Scheme {
Seafile,
/// [Upyun][crate::services::Upyun]: Upyun Services.
Upyun,
+ /// [Chainsafe][crate::services::Upyun]: Upyun Services.
Review Comment:
fixed
##########
core/src/services/chainsafe/backend.rs:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::services::chainsafe::core::ChainsafeSigner;
+use crate::*;
+
+use super::core::parse_info;
+use super::core::ChainsafeCore;
+use super::core::ObjectInfoResponse;
+use super::error::parse_error;
+use super::lister::ChainsafeLister;
+use super::writer::ChainsafeWriter;
+use super::writer::ChainsafeWriters;
+
+/// Config for backblaze Chainsafe services support.
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct ChainsafeConfig {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+ /// secret of this backend.
+ pub secret: Option<String>,
+ /// bucket of this backend.
+ ///
+ /// required.
+ pub bucket: String,
+}
+
+impl Debug for ChainsafeConfig {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ let mut d = f.debug_struct("ChainsafeConfig");
+
+ d.field("root", &self.root).field("bucket", &self.bucket);
+
+ d.finish_non_exhaustive()
+ }
+}
+
+/// [chainsafe](https://storage.chainsafe.io/) services support.
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct ChainsafeBuilder {
+ config: ChainsafeConfig,
+
+ http_client: Option<HttpClient>,
+}
+
+impl Debug for ChainsafeBuilder {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ let mut d = f.debug_struct("ChainsafeBuilder");
+
+ d.field("config", &self.config);
+ d.finish_non_exhaustive()
+ }
+}
+
+impl ChainsafeBuilder {
+ /// Set root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub fn root(&mut self, root: &str) -> &mut Self {
+ self.config.root = if root.is_empty() {
+ None
+ } else {
+ Some(root.to_string())
+ };
+
+ self
+ }
+
+ /// secret of this backend.
+ ///
+ /// required.
+ pub fn secret(&mut self, secret: &str) -> &mut Self {
+ self.config.secret = if secret.is_empty() {
+ None
+ } else {
+ Some(secret.to_string())
+ };
+
+ self
+ }
+
+ /// Set bucket name of this backend.
+ pub fn bucket(&mut self, bucket: &str) -> &mut Self {
+ self.config.bucket = bucket.to_string();
+
+ self
+ }
+
+ /// Specify the http client that used by this service.
+ ///
+ /// # Notes
+ ///
+ /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed
+ /// during minor updates.
+ pub fn http_client(&mut self, client: HttpClient) -> &mut Self {
+ self.http_client = Some(client);
+ self
+ }
+}
+
+impl Builder for ChainsafeBuilder {
+ const SCHEME: Scheme = Scheme::Chainsafe;
+ type Accessor = ChainsafeBackend;
+
+ /// Converts a HashMap into an ChainsafeBuilder instance.
+ ///
+ /// # Arguments
+ ///
+ /// * `map` - A HashMap containing the configuration values.
+ ///
+ /// # Returns
+ ///
+ /// Returns an instance of ChainsafeBuilder.
+ fn from_map(map: HashMap<String, String>) -> Self {
+ // Deserialize the configuration from the HashMap.
+ let config = ChainsafeConfig::deserialize(ConfigDeserializer::new(map))
+ .expect("config deserialize must succeed");
+
+ // Create an ChainsafeBuilder instance with the deserialized config.
+ ChainsafeBuilder {
+ config,
+ http_client: None,
+ }
+ }
+
+ /// Builds the backend and returns the result of ChainsafeBackend.
+ fn build(&mut self) -> Result<Self::Accessor> {
+ debug!("backend build started: {:?}", &self);
+
+ let root = normalize_root(&self.config.root.clone().unwrap_or_default());
+ debug!("backend use root {}", &root);
+
+ // Handle bucket.
+ if self.config.bucket.is_empty() {
+ return Err(Error::new(ErrorKind::ConfigInvalid, "bucket is empty")
+ .with_operation("Builder::build")
+ .with_context("service", Scheme::Chainsafe));
+ }
+
+ debug!("backend use bucket {}", &self.config.bucket);
+
+ let secret = match &self.config.secret {
+ Some(secret) => Ok(secret.clone()),
+ None => Err(
+ Error::new(ErrorKind::ConfigInvalid, "application_key_id is empty")
+ .with_operation("Builder::build")
+ .with_context("service", Scheme::Chainsafe),
+ ),
+ }?;
+
+ let client = if let Some(client) = self.http_client.take() {
+ client
+ } else {
+ HttpClient::new().map_err(|err| {
+ err.with_operation("Builder::build")
+ .with_context("service", Scheme::Chainsafe)
+ })?
+ };
+
+ let signer = ChainsafeSigner::default();
+
+ Ok(ChainsafeBackend {
+ core: Arc::new(ChainsafeCore {
+ signer: Arc::new(RwLock::new(signer)),
+ root,
+ secret,
+ bucket: self.config.bucket.clone(),
+ client,
+ }),
+ })
+ }
+}
+
+/// Backend for Chainsafe services.
+#[derive(Debug, Clone)]
+pub struct ChainsafeBackend {
+ core: Arc<ChainsafeCore>,
+}
+
+#[async_trait]
+impl Accessor for ChainsafeBackend {
+ type Reader = IncomingAsyncBody;
+
+ type BlockingReader = ();
+
+ type Writer = ChainsafeWriters;
+
+ type BlockingWriter = ();
+
+ type Lister = oio::PageLister<ChainsafeLister>;
+
+ type BlockingLister = ();
+
+ fn info(&self) -> AccessorInfo {
+ let mut am = AccessorInfo::default();
+ am.set_scheme(Scheme::Chainsafe)
+ .set_root(&self.core.root)
+ .set_native_capability(Capability {
+ stat: true,
+
+ read: true,
+
+ create_dir: true,
+ write: true,
+ write_can_empty: true,
+
+ delete: true,
+
+ list: true,
+
+ ..Default::default()
+ });
+
+ am
+ }
+
+ async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
+ let resp = self.core.create_dir(path).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => Ok(RpCreateDir::default()),
+ // Allow 409 when creating a existing dir
+ StatusCode::CONFLICT => Ok(RpCreateDir::default()),
+ _ => Err(parse_error(resp).await?),
+ }
+ }
+
+ async fn read(&self, path: &str, _args: OpRead) -> Result<(RpRead, Self::Reader)> {
+ let resp = self.core.download_object(path).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let size = parse_content_length(resp.headers())?;
+ let range = parse_content_range(resp.headers())?;
+ Ok((
+ RpRead::new().with_size(size).with_range(range),
+ resp.into_body(),
+ ))
+ }
+ _ => Err(parse_error(resp).await?),
+ }
+ }
+
+ async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> {
+ let resp = self.core.object_info(path).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: ObjectInfoResponse =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436994905
##########
core/src/services/chainsafe/backend.rs:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::services::chainsafe::core::ChainsafeSigner;
+use crate::*;
+
+use super::core::parse_info;
+use super::core::ChainsafeCore;
+use super::core::ObjectInfoResponse;
+use super::error::parse_error;
+use super::lister::ChainsafeLister;
+use super::writer::ChainsafeWriter;
+use super::writer::ChainsafeWriters;
+
+/// Config for backblaze Chainsafe services support.
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct ChainsafeConfig {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+ /// secret of this backend.
+ pub secret: Option<String>,
Review Comment:
got it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437029376
##########
core/src/services/chainsafe/core.rs:
##########
@@ -0,0 +1,373 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use http::header;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+use serde_json::json;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::*;
+
+use super::error::parse_error;
+
+/// Core of [chainsafe](https://storage.chainsafe.io/) services support.
+#[derive(Clone)]
+pub struct ChainsafeCore {
+ pub signer: Arc<RwLock<ChainsafeSigner>>,
+
+ /// The root of this core.
+ pub root: String,
+ /// The api_secret of this core.
+ pub api_secret: String,
+ /// The bucket name of this backend.
+ pub bucket: String,
+
+ pub client: HttpClient,
+}
+
+impl Debug for ChainsafeCore {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Backend")
+ .field("root", &self.root)
+ .field("bucket", &self.bucket)
+ .finish_non_exhaustive()
+ }
+}
+
+impl ChainsafeCore {
+ #[inline]
+ pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> {
+ self.client.send(req).await
+ }
+
+ pub async fn get_auth_info(&self) -> Result<AuthInfo> {
+ {
+ let signer = self.signer.read().await;
+
+ if !signer.auth_info.bucket_id.is_empty() {
+ let auth_info = signer.auth_info.clone();
+ return Ok(auth_info);
+ }
+ }
+
+ {
+ let mut signer = self.signer.write().await;
+
+ if !signer.auth_info.bucket_id.is_empty() {
+ let auth_info = signer.auth_info.clone();
+ return Ok(auth_info);
+ }
+
+ let req = Request::get("https://api.chainsafe.io/api/v1/buckets")
+ .header(
+ header::AUTHORIZATION,
+ format_authorization_by_bearer(&self.api_secret)?,
+ )
+ .body(AsyncBody::Empty)
+ .map_err(new_request_build_error)?;
+
+ let resp = self.send(req).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let resp_body = &resp.into_body().bytes().await?;
+
+ let buckets: Vec<Bucket> =
+ serde_json::from_slice(resp_body).map_err(new_json_deserialize_error)?;
+
+ for bucket in buckets {
+ if let Some(name) = bucket.name {
+ if self.bucket == name {
+ signer.auth_info.bucket_id = bucket.id;
+ break;
+ }
+ }
+ }
+
+ // bucket not found
+ if signer.auth_info.bucket_id.is_empty() {
Review Comment:
> I don't have any objections, but is it acceptable to call the 'list buckets' API for the bucket_id?
No, I don't like this idea; it increases our maintenance burden. I'm ok to add it back in the future if there are real users complain about this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437304659
##########
core/src/services/chainsafe/lister.rs:
##########
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::{parse_info, ChainsafeCore, Info};
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct ChainsafeLister {
+ core: Arc<ChainsafeCore>,
+ path: String,
+}
+
+impl ChainsafeLister {
+ pub fn new(core: Arc<ChainsafeCore>, path: &str) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ }
+ }
+}
+
+#[async_trait]
+impl oio::PageList for ChainsafeLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let resp = self.core.list_objects(&self.path).await?;
+
+ match resp.status() {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: Vec<Info> =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ for info in output {
+ let mut path = if self.path.starts_with('/') {
Review Comment:
> I understand that path and root mean the same thing in this scenario
The term `root` refers to the base of this service, which has been standardized by `normalize_root`. However, it's not a significant problem. Address it using your preferred methods.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436975941
##########
core/src/services/chainsafe/backend.rs:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::services::chainsafe::core::ChainsafeSigner;
+use crate::*;
+
+use super::core::parse_info;
+use super::core::ChainsafeCore;
+use super::core::ObjectInfoResponse;
+use super::error::parse_error;
+use super::lister::ChainsafeLister;
+use super::writer::ChainsafeWriter;
+use super::writer::ChainsafeWriters;
+
+/// Config for backblaze Chainsafe services support.
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct ChainsafeConfig {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+ /// secret of this backend.
+ pub secret: Option<String>,
Review Comment:
we do not need key, only need secret
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437033079
##########
core/src/services/chainsafe/lister.rs:
##########
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::{parse_info, ChainsafeCore, Info};
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct ChainsafeLister {
+ core: Arc<ChainsafeCore>,
+ path: String,
+}
+
+impl ChainsafeLister {
+ pub fn new(core: Arc<ChainsafeCore>, path: &str) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ }
+ }
+}
+
+#[async_trait]
+impl oio::PageList for ChainsafeLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let resp = self.core.list_objects(&self.path).await?;
+
+ match resp.status() {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: Vec<Info> =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ for info in output {
+ let mut path = if self.path.starts_with('/') {
Review Comment:
By the way, `build_abs_path` is designed to play with `root`. It's undefined behavior when you put a `path` in it directly. I'm fine to use `format!("{path}/name").trim_start_matches('/')` here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437350126
##########
core/src/services/chainsafe/docs.md:
##########
@@ -0,0 +1,50 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [ ] rename
+- [x] list
+- [x] scan
+- [ ] presign
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the work directory for backend
+- `bucket_id` Chainsafe bucket_id
+- `secret` Chainsafe secret
+
+You can refer to [`ChainsafeBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust
+use anyhow::Result;
+use opendal::services::Chainsafe;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ // create backend builder
+ let mut builder = Chainsafe::default();
+
+ // set the storage root for OpenDAL
+ builder.root("/");
+ // set the bucket_id for OpenDAL
+ builder.bucket_id("opendal");
+ // set the secret for OpenDAL
+ builder.secret("xxxxxxxxxxxxx");
Review Comment:
Seems not updated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436986019
##########
core/src/services/chainsafe/backend.rs:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::services::chainsafe::core::ChainsafeSigner;
+use crate::*;
+
+use super::core::parse_info;
+use super::core::ChainsafeCore;
+use super::core::ObjectInfoResponse;
+use super::error::parse_error;
+use super::lister::ChainsafeLister;
+use super::writer::ChainsafeWriter;
+use super::writer::ChainsafeWriters;
+
+/// Config for backblaze Chainsafe services support.
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct ChainsafeConfig {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+ /// secret of this backend.
+ pub secret: Option<String>,
Review Comment:
Is it an `api key`? This could be confusing for users who might mix it up with the S3 key, which also has a secret.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437028321
##########
core/src/services/chainsafe/lister.rs:
##########
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::{parse_info, ChainsafeCore, Info};
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct ChainsafeLister {
+ core: Arc<ChainsafeCore>,
+ path: String,
+}
+
+impl ChainsafeLister {
+ pub fn new(core: Arc<ChainsafeCore>, path: &str) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ }
+ }
+}
+
+#[async_trait]
+impl oio::PageList for ChainsafeLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let resp = self.core.list_objects(&self.path).await?;
+
+ match resp.status() {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: Vec<Info> =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ for info in output {
+ let mut path = if self.path.starts_with('/') {
Review Comment:
> Perhaps I should call `normalize_root` during initialization?
Yep.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437060201
##########
core/src/services/chainsafe/lister.rs:
##########
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::{parse_info, ChainsafeCore, Info};
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct ChainsafeLister {
+ core: Arc<ChainsafeCore>,
+ path: String,
+}
+
+impl ChainsafeLister {
+ pub fn new(core: Arc<ChainsafeCore>, path: &str) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ }
+ }
+}
+
+#[async_trait]
+impl oio::PageList for ChainsafeLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let resp = self.core.list_objects(&self.path).await?;
+
+ match resp.status() {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: Vec<Info> =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ for info in output {
+ let mut path = if self.path.starts_with('/') {
Review Comment:
I understand that path and root mean the same thing in this scenario
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436996452
##########
core/src/services/chainsafe/core.rs:
##########
@@ -0,0 +1,373 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use http::header;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+use serde_json::json;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::*;
+
+use super::error::parse_error;
+
+/// Core of [chainsafe](https://storage.chainsafe.io/) services support.
+#[derive(Clone)]
+pub struct ChainsafeCore {
+ pub signer: Arc<RwLock<ChainsafeSigner>>,
+
+ /// The root of this core.
+ pub root: String,
+ /// The api_secret of this core.
+ pub api_secret: String,
+ /// The bucket name of this backend.
+ pub bucket: String,
+
+ pub client: HttpClient,
+}
+
+impl Debug for ChainsafeCore {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Backend")
+ .field("root", &self.root)
+ .field("bucket", &self.bucket)
+ .finish_non_exhaustive()
+ }
+}
+
+impl ChainsafeCore {
+ #[inline]
+ pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> {
+ self.client.send(req).await
+ }
+
+ pub async fn get_auth_info(&self) -> Result<AuthInfo> {
+ {
+ let signer = self.signer.read().await;
+
+ if !signer.auth_info.bucket_id.is_empty() {
+ let auth_info = signer.auth_info.clone();
+ return Ok(auth_info);
+ }
+ }
+
+ {
+ let mut signer = self.signer.write().await;
+
+ if !signer.auth_info.bucket_id.is_empty() {
+ let auth_info = signer.auth_info.clone();
+ return Ok(auth_info);
+ }
+
+ let req = Request::get("https://api.chainsafe.io/api/v1/buckets")
+ .header(
+ header::AUTHORIZATION,
+ format_authorization_by_bearer(&self.api_secret)?,
+ )
+ .body(AsyncBody::Empty)
+ .map_err(new_request_build_error)?;
+
+ let resp = self.send(req).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let resp_body = &resp.into_body().bytes().await?;
+
+ let buckets: Vec<Bucket> =
+ serde_json::from_slice(resp_body).map_err(new_json_deserialize_error)?;
+
+ for bucket in buckets {
+ if let Some(name) = bucket.name {
+ if self.bucket == name {
+ signer.auth_info.bucket_id = bucket.id;
+ break;
+ }
+ }
+ }
+
+ // bucket not found
+ if signer.auth_info.bucket_id.is_empty() {
Review Comment:
To make everything simpler, how about enforcing users to input bucket_id directly?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436992178
##########
core/src/services/chainsafe/backend.rs:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::services::chainsafe::core::ChainsafeSigner;
Review Comment:
fixed
##########
.env.example:
##########
@@ -190,4 +190,8 @@ OPENDAL_SEAFILE_REPO_NAME=<repo_name>
OPENDAL_UPYUN_ROOT=/path/to/dir
OPENDAL_UPYUN_BUCKET=<bucket>
OPENDAL_UPYUN_OPERATOR=<operator>
-OPENDAL_UPYUN_PASSWORD=<password>
\ No newline at end of file
+OPENDAL_UPYUN_PASSWORD=<password>
+# chainsafe
+OPENDAL_UPYUN_ROOT=/path/to/dir
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436992608
##########
core/src/services/chainsafe/backend.rs:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::services::chainsafe::core::ChainsafeSigner;
+use crate::*;
+
+use super::core::parse_info;
+use super::core::ChainsafeCore;
+use super::core::ObjectInfoResponse;
+use super::error::parse_error;
+use super::lister::ChainsafeLister;
+use super::writer::ChainsafeWriter;
+use super::writer::ChainsafeWriters;
+
+/// Config for backblaze Chainsafe services support.
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct ChainsafeConfig {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+ /// secret of this backend.
+ pub secret: Option<String>,
Review Comment:
> Yes, maybe I should name it api_secret? We do not need key id, only need secret
I prefer keep `api_key` only. It does be an api key.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436975941
##########
core/src/services/chainsafe/backend.rs:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::services::chainsafe::core::ChainsafeSigner;
+use crate::*;
+
+use super::core::parse_info;
+use super::core::ChainsafeCore;
+use super::core::ObjectInfoResponse;
+use super::error::parse_error;
+use super::lister::ChainsafeLister;
+use super::writer::ChainsafeWriter;
+use super::writer::ChainsafeWriters;
+
+/// Config for backblaze Chainsafe services support.
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct ChainsafeConfig {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+ /// secret of this backend.
+ pub secret: Option<String>,
Review Comment:
we do not need key, only need secret
<img width="579" alt="image" src="https://github.com/apache/incubator-opendal/assets/26793951/cecef0be-b59a-45f0-ae52-50637a8b7d4f">
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437030611
##########
core/src/services/chainsafe/core.rs:
##########
@@ -0,0 +1,373 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use http::header;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+use serde_json::json;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::*;
+
+use super::error::parse_error;
+
+/// Core of [chainsafe](https://storage.chainsafe.io/) services support.
+#[derive(Clone)]
+pub struct ChainsafeCore {
+ pub signer: Arc<RwLock<ChainsafeSigner>>,
+
+ /// The root of this core.
+ pub root: String,
+ /// The api_secret of this core.
+ pub api_secret: String,
+ /// The bucket name of this backend.
+ pub bucket: String,
+
+ pub client: HttpClient,
+}
+
+impl Debug for ChainsafeCore {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Backend")
+ .field("root", &self.root)
+ .field("bucket", &self.bucket)
+ .finish_non_exhaustive()
+ }
+}
+
+impl ChainsafeCore {
+ #[inline]
+ pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> {
+ self.client.send(req).await
+ }
+
+ pub async fn get_auth_info(&self) -> Result<AuthInfo> {
+ {
+ let signer = self.signer.read().await;
+
+ if !signer.auth_info.bucket_id.is_empty() {
+ let auth_info = signer.auth_info.clone();
+ return Ok(auth_info);
+ }
+ }
+
+ {
+ let mut signer = self.signer.write().await;
+
+ if !signer.auth_info.bucket_id.is_empty() {
+ let auth_info = signer.auth_info.clone();
+ return Ok(auth_info);
+ }
+
+ let req = Request::get("https://api.chainsafe.io/api/v1/buckets")
+ .header(
+ header::AUTHORIZATION,
+ format_authorization_by_bearer(&self.api_secret)?,
+ )
+ .body(AsyncBody::Empty)
+ .map_err(new_request_build_error)?;
+
+ let resp = self.send(req).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let resp_body = &resp.into_body().bytes().await?;
+
+ let buckets: Vec<Bucket> =
+ serde_json::from_slice(resp_body).map_err(new_json_deserialize_error)?;
+
+ for bucket in buckets {
+ if let Some(name) = bucket.name {
+ if self.bucket == name {
+ signer.auth_info.bucket_id = bucket.id;
+ break;
+ }
+ }
+ }
+
+ // bucket not found
+ if signer.auth_info.bucket_id.is_empty() {
Review Comment:
So are we switching to directly inputting the bucket_id?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436997137
##########
core/src/services/chainsafe/lister.rs:
##########
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::{parse_info, ChainsafeCore, Info};
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct ChainsafeLister {
+ core: Arc<ChainsafeCore>,
+ path: String,
+}
+
+impl ChainsafeLister {
+ pub fn new(core: Arc<ChainsafeCore>, path: &str) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ }
+ }
+}
+
+#[async_trait]
+impl oio::PageList for ChainsafeLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let resp = self.core.list_objects(&self.path).await?;
+
+ match resp.status() {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: Vec<Info> =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ for info in output {
+ let mut path = if self.path.starts_with('/') {
Review Comment:
I didn't get this. All `path` input from `list` has been normalized, so it must be start with `/`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436987688
##########
core/src/services/chainsafe/backend.rs:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::services::chainsafe::core::ChainsafeSigner;
+use crate::*;
+
+use super::core::parse_info;
+use super::core::ChainsafeCore;
+use super::core::ObjectInfoResponse;
+use super::error::parse_error;
+use super::lister::ChainsafeLister;
+use super::writer::ChainsafeWriter;
+use super::writer::ChainsafeWriters;
+
+/// Config for backblaze Chainsafe services support.
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct ChainsafeConfig {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+ /// secret of this backend.
+ pub secret: Option<String>,
Review Comment:
Yes, maybe I should name it api_secret? We do not need key id, only need secret
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436987688
##########
core/src/services/chainsafe/backend.rs:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::services::chainsafe::core::ChainsafeSigner;
+use crate::*;
+
+use super::core::parse_info;
+use super::core::ChainsafeCore;
+use super::core::ObjectInfoResponse;
+use super::error::parse_error;
+use super::lister::ChainsafeLister;
+use super::writer::ChainsafeWriter;
+use super::writer::ChainsafeWriters;
+
+/// Config for backblaze Chainsafe services support.
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct ChainsafeConfig {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+ /// secret of this backend.
+ pub secret: Option<String>,
Review Comment:
Yes, maybe I should name it api_secret?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437123108
##########
core/src/services/chainsafe/core.rs:
##########
@@ -0,0 +1,373 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use http::header;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+use serde_json::json;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::*;
+
+use super::error::parse_error;
+
+/// Core of [chainsafe](https://storage.chainsafe.io/) services support.
+#[derive(Clone)]
+pub struct ChainsafeCore {
+ pub signer: Arc<RwLock<ChainsafeSigner>>,
+
+ /// The root of this core.
+ pub root: String,
+ /// The api_secret of this core.
+ pub api_secret: String,
+ /// The bucket name of this backend.
+ pub bucket: String,
+
+ pub client: HttpClient,
+}
+
+impl Debug for ChainsafeCore {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Backend")
+ .field("root", &self.root)
+ .field("bucket", &self.bucket)
+ .finish_non_exhaustive()
+ }
+}
+
+impl ChainsafeCore {
+ #[inline]
+ pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> {
+ self.client.send(req).await
+ }
+
+ pub async fn get_auth_info(&self) -> Result<AuthInfo> {
+ {
+ let signer = self.signer.read().await;
+
+ if !signer.auth_info.bucket_id.is_empty() {
+ let auth_info = signer.auth_info.clone();
+ return Ok(auth_info);
+ }
+ }
+
+ {
+ let mut signer = self.signer.write().await;
+
+ if !signer.auth_info.bucket_id.is_empty() {
+ let auth_info = signer.auth_info.clone();
+ return Ok(auth_info);
+ }
+
+ let req = Request::get("https://api.chainsafe.io/api/v1/buckets")
+ .header(
+ header::AUTHORIZATION,
+ format_authorization_by_bearer(&self.api_secret)?,
+ )
+ .body(AsyncBody::Empty)
+ .map_err(new_request_build_error)?;
+
+ let resp = self.send(req).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let resp_body = &resp.into_body().bytes().await?;
+
+ let buckets: Vec<Bucket> =
+ serde_json::from_slice(resp_body).map_err(new_json_deserialize_error)?;
+
+ for bucket in buckets {
+ if let Some(name) = bucket.name {
+ if self.bucket == name {
+ signer.auth_info.bucket_id = bucket.id;
+ break;
+ }
+ }
+ }
+
+ // bucket not found
+ if signer.auth_info.bucket_id.is_empty() {
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436983045
##########
core/src/services/chainsafe/lister.rs:
##########
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::{parse_info, ChainsafeCore, Info};
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct ChainsafeLister {
+ core: Arc<ChainsafeCore>,
+ path: String,
+}
+
+impl ChainsafeLister {
+ pub fn new(core: Arc<ChainsafeCore>, path: &str) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ }
+ }
+}
+
+#[async_trait]
+impl oio::PageList for ChainsafeLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let resp = self.core.list_objects(&self.path).await?;
+
+ match resp.status() {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: Vec<Info> =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ for info in output {
+ let mut path = if self.path.starts_with('/') {
Review Comment:
Because build_abs_path need root be start with /, so we can return dir path as xxx/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437024672
##########
core/src/services/chainsafe/core.rs:
##########
@@ -0,0 +1,373 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use http::header;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+use serde_json::json;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::*;
+
+use super::error::parse_error;
+
+/// Core of [chainsafe](https://storage.chainsafe.io/) services support.
+#[derive(Clone)]
+pub struct ChainsafeCore {
+ pub signer: Arc<RwLock<ChainsafeSigner>>,
+
+ /// The root of this core.
+ pub root: String,
+ /// The api_secret of this core.
+ pub api_secret: String,
+ /// The bucket name of this backend.
+ pub bucket: String,
+
+ pub client: HttpClient,
+}
+
+impl Debug for ChainsafeCore {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Backend")
+ .field("root", &self.root)
+ .field("bucket", &self.bucket)
+ .finish_non_exhaustive()
+ }
+}
+
+impl ChainsafeCore {
+ #[inline]
+ pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> {
+ self.client.send(req).await
+ }
+
+ pub async fn get_auth_info(&self) -> Result<AuthInfo> {
+ {
+ let signer = self.signer.read().await;
+
+ if !signer.auth_info.bucket_id.is_empty() {
+ let auth_info = signer.auth_info.clone();
+ return Ok(auth_info);
+ }
+ }
+
+ {
+ let mut signer = self.signer.write().await;
+
+ if !signer.auth_info.bucket_id.is_empty() {
+ let auth_info = signer.auth_info.clone();
+ return Ok(auth_info);
+ }
+
+ let req = Request::get("https://api.chainsafe.io/api/v1/buckets")
+ .header(
+ header::AUTHORIZATION,
+ format_authorization_by_bearer(&self.api_secret)?,
+ )
+ .body(AsyncBody::Empty)
+ .map_err(new_request_build_error)?;
+
+ let resp = self.send(req).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let resp_body = &resp.into_body().bytes().await?;
+
+ let buckets: Vec<Bucket> =
+ serde_json::from_slice(resp_body).map_err(new_json_deserialize_error)?;
+
+ for bucket in buckets {
+ if let Some(name) = bucket.name {
+ if self.bucket == name {
+ signer.auth_info.bucket_id = bucket.id;
+ break;
+ }
+ }
+ }
+
+ // bucket not found
+ if signer.auth_info.bucket_id.is_empty() {
Review Comment:
I don't have any objections, but is it acceptable to call the 'list buckets' API for the bucket_id?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo merged PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437033300
##########
core/src/services/chainsafe/core.rs:
##########
@@ -0,0 +1,373 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use http::header;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+use serde_json::json;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::*;
+
+use super::error::parse_error;
+
+/// Core of [chainsafe](https://storage.chainsafe.io/) services support.
+#[derive(Clone)]
+pub struct ChainsafeCore {
+ pub signer: Arc<RwLock<ChainsafeSigner>>,
+
+ /// The root of this core.
+ pub root: String,
+ /// The api_secret of this core.
+ pub api_secret: String,
+ /// The bucket name of this backend.
+ pub bucket: String,
+
+ pub client: HttpClient,
+}
+
+impl Debug for ChainsafeCore {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Backend")
+ .field("root", &self.root)
+ .field("bucket", &self.bucket)
+ .finish_non_exhaustive()
+ }
+}
+
+impl ChainsafeCore {
+ #[inline]
+ pub async fn send(&self, req: Request<AsyncBody>) -> Result<Response<IncomingAsyncBody>> {
+ self.client.send(req).await
+ }
+
+ pub async fn get_auth_info(&self) -> Result<AuthInfo> {
+ {
+ let signer = self.signer.read().await;
+
+ if !signer.auth_info.bucket_id.is_empty() {
+ let auth_info = signer.auth_info.clone();
+ return Ok(auth_info);
+ }
+ }
+
+ {
+ let mut signer = self.signer.write().await;
+
+ if !signer.auth_info.bucket_id.is_empty() {
+ let auth_info = signer.auth_info.clone();
+ return Ok(auth_info);
+ }
+
+ let req = Request::get("https://api.chainsafe.io/api/v1/buckets")
+ .header(
+ header::AUTHORIZATION,
+ format_authorization_by_bearer(&self.api_secret)?,
+ )
+ .body(AsyncBody::Empty)
+ .map_err(new_request_build_error)?;
+
+ let resp = self.send(req).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let resp_body = &resp.into_body().bytes().await?;
+
+ let buckets: Vec<Bucket> =
+ serde_json::from_slice(resp_body).map_err(new_json_deserialize_error)?;
+
+ for bucket in buckets {
+ if let Some(name) = bucket.name {
+ if self.bucket == name {
+ signer.auth_info.bucket_id = bucket.id;
+ break;
+ }
+ }
+ }
+
+ // bucket not found
+ if signer.auth_info.bucket_id.is_empty() {
Review Comment:
> So are we switching to directly inputting the bucket_id?
Yes, let's do this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "Xuanwo (via GitHub)" <gi...@apache.org>.
Xuanwo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436952172
##########
core/src/services/chainsafe/backend.rs:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::services::chainsafe::core::ChainsafeSigner;
Review Comment:
plese use `super::core::*` instead.
##########
core/src/services/chainsafe/lister.rs:
##########
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::{parse_info, ChainsafeCore, Info};
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct ChainsafeLister {
+ core: Arc<ChainsafeCore>,
+ path: String,
+}
+
+impl ChainsafeLister {
+ pub fn new(core: Arc<ChainsafeCore>, path: &str) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ }
+ }
+}
+
+#[async_trait]
+impl oio::PageList for ChainsafeLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let resp = self.core.list_objects(&self.path).await?;
+
+ match resp.status() {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: Vec<Info> =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ for info in output {
+ let mut path = if self.path.starts_with('/') {
Review Comment:
Why we need to check `self.path` like this?
##########
core/src/services/chainsafe/backend.rs:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::services::chainsafe::core::ChainsafeSigner;
+use crate::*;
+
+use super::core::parse_info;
+use super::core::ChainsafeCore;
+use super::core::ObjectInfoResponse;
+use super::error::parse_error;
+use super::lister::ChainsafeLister;
+use super::writer::ChainsafeWriter;
+use super::writer::ChainsafeWriters;
+
+/// Config for backblaze Chainsafe services support.
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct ChainsafeConfig {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+ /// secret of this backend.
+ pub secret: Option<String>,
+ /// bucket of this backend.
+ ///
+ /// required.
+ pub bucket: String,
+}
+
+impl Debug for ChainsafeConfig {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ let mut d = f.debug_struct("ChainsafeConfig");
+
+ d.field("root", &self.root).field("bucket", &self.bucket);
+
+ d.finish_non_exhaustive()
+ }
+}
+
+/// [chainsafe](https://storage.chainsafe.io/) services support.
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct ChainsafeBuilder {
+ config: ChainsafeConfig,
+
+ http_client: Option<HttpClient>,
+}
+
+impl Debug for ChainsafeBuilder {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ let mut d = f.debug_struct("ChainsafeBuilder");
+
+ d.field("config", &self.config);
+ d.finish_non_exhaustive()
+ }
+}
+
+impl ChainsafeBuilder {
+ /// Set root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub fn root(&mut self, root: &str) -> &mut Self {
+ self.config.root = if root.is_empty() {
+ None
+ } else {
+ Some(root.to_string())
+ };
+
+ self
+ }
+
+ /// secret of this backend.
+ ///
+ /// required.
+ pub fn secret(&mut self, secret: &str) -> &mut Self {
+ self.config.secret = if secret.is_empty() {
+ None
+ } else {
+ Some(secret.to_string())
+ };
+
+ self
+ }
+
+ /// Set bucket name of this backend.
+ pub fn bucket(&mut self, bucket: &str) -> &mut Self {
+ self.config.bucket = bucket.to_string();
+
+ self
+ }
+
+ /// Specify the http client that used by this service.
+ ///
+ /// # Notes
+ ///
+ /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed
+ /// during minor updates.
+ pub fn http_client(&mut self, client: HttpClient) -> &mut Self {
+ self.http_client = Some(client);
+ self
+ }
+}
+
+impl Builder for ChainsafeBuilder {
+ const SCHEME: Scheme = Scheme::Chainsafe;
+ type Accessor = ChainsafeBackend;
+
+ /// Converts a HashMap into an ChainsafeBuilder instance.
+ ///
+ /// # Arguments
+ ///
+ /// * `map` - A HashMap containing the configuration values.
+ ///
+ /// # Returns
+ ///
+ /// Returns an instance of ChainsafeBuilder.
+ fn from_map(map: HashMap<String, String>) -> Self {
+ // Deserialize the configuration from the HashMap.
+ let config = ChainsafeConfig::deserialize(ConfigDeserializer::new(map))
+ .expect("config deserialize must succeed");
+
+ // Create an ChainsafeBuilder instance with the deserialized config.
+ ChainsafeBuilder {
+ config,
+ http_client: None,
+ }
+ }
+
+ /// Builds the backend and returns the result of ChainsafeBackend.
+ fn build(&mut self) -> Result<Self::Accessor> {
+ debug!("backend build started: {:?}", &self);
+
+ let root = normalize_root(&self.config.root.clone().unwrap_or_default());
+ debug!("backend use root {}", &root);
+
+ // Handle bucket.
+ if self.config.bucket.is_empty() {
+ return Err(Error::new(ErrorKind::ConfigInvalid, "bucket is empty")
+ .with_operation("Builder::build")
+ .with_context("service", Scheme::Chainsafe));
+ }
+
+ debug!("backend use bucket {}", &self.config.bucket);
+
+ let secret = match &self.config.secret {
+ Some(secret) => Ok(secret.clone()),
+ None => Err(
+ Error::new(ErrorKind::ConfigInvalid, "application_key_id is empty")
+ .with_operation("Builder::build")
+ .with_context("service", Scheme::Chainsafe),
+ ),
+ }?;
+
+ let client = if let Some(client) = self.http_client.take() {
+ client
+ } else {
+ HttpClient::new().map_err(|err| {
+ err.with_operation("Builder::build")
+ .with_context("service", Scheme::Chainsafe)
+ })?
+ };
+
+ let signer = ChainsafeSigner::default();
+
+ Ok(ChainsafeBackend {
+ core: Arc::new(ChainsafeCore {
+ signer: Arc::new(RwLock::new(signer)),
+ root,
+ secret,
+ bucket: self.config.bucket.clone(),
+ client,
+ }),
+ })
+ }
+}
+
+/// Backend for Chainsafe services.
+#[derive(Debug, Clone)]
+pub struct ChainsafeBackend {
+ core: Arc<ChainsafeCore>,
+}
+
+#[async_trait]
+impl Accessor for ChainsafeBackend {
+ type Reader = IncomingAsyncBody;
+
+ type BlockingReader = ();
+
+ type Writer = ChainsafeWriters;
+
+ type BlockingWriter = ();
+
+ type Lister = oio::PageLister<ChainsafeLister>;
+
+ type BlockingLister = ();
+
+ fn info(&self) -> AccessorInfo {
+ let mut am = AccessorInfo::default();
+ am.set_scheme(Scheme::Chainsafe)
+ .set_root(&self.core.root)
+ .set_native_capability(Capability {
+ stat: true,
+
+ read: true,
+
+ create_dir: true,
+ write: true,
+ write_can_empty: true,
+
+ delete: true,
+
+ list: true,
+
+ ..Default::default()
+ });
+
+ am
+ }
+
+ async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
+ let resp = self.core.create_dir(path).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => Ok(RpCreateDir::default()),
+ // Allow 409 when creating a existing dir
+ StatusCode::CONFLICT => Ok(RpCreateDir::default()),
+ _ => Err(parse_error(resp).await?),
+ }
+ }
+
+ async fn read(&self, path: &str, _args: OpRead) -> Result<(RpRead, Self::Reader)> {
+ let resp = self.core.download_object(path).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let size = parse_content_length(resp.headers())?;
+ let range = parse_content_range(resp.headers())?;
+ Ok((
+ RpRead::new().with_size(size).with_range(range),
+ resp.into_body(),
+ ))
+ }
+ _ => Err(parse_error(resp).await?),
+ }
+ }
+
+ async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> {
+ let resp = self.core.object_info(path).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: ObjectInfoResponse =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
Review Comment:
Please use `serde_json::from_slice` instead.
##########
core/src/types/scheme.rs:
##########
@@ -44,6 +44,8 @@ pub enum Scheme {
Seafile,
/// [Upyun][crate::services::Upyun]: Upyun Services.
Upyun,
+ /// [Chainsafe][crate::services::Upyun]: Upyun Services.
Review Comment:
typos.
##########
core/src/services/chainsafe/backend.rs:
##########
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use tokio::sync::RwLock;
+
+use crate::raw::*;
+use crate::services::chainsafe::core::ChainsafeSigner;
+use crate::*;
+
+use super::core::parse_info;
+use super::core::ChainsafeCore;
+use super::core::ObjectInfoResponse;
+use super::error::parse_error;
+use super::lister::ChainsafeLister;
+use super::writer::ChainsafeWriter;
+use super::writer::ChainsafeWriters;
+
+/// Config for backblaze Chainsafe services support.
+#[derive(Default, Deserialize)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct ChainsafeConfig {
+ /// root of this backend.
+ ///
+ /// All operations will happen under this root.
+ pub root: Option<String>,
+ /// secret of this backend.
+ pub secret: Option<String>,
Review Comment:
Seems chainsafe is using `api_key`?
##########
.env.example:
##########
@@ -190,4 +190,8 @@ OPENDAL_SEAFILE_REPO_NAME=<repo_name>
OPENDAL_UPYUN_ROOT=/path/to/dir
OPENDAL_UPYUN_BUCKET=<bucket>
OPENDAL_UPYUN_OPERATOR=<operator>
-OPENDAL_UPYUN_PASSWORD=<password>
\ No newline at end of file
+OPENDAL_UPYUN_PASSWORD=<password>
+# chainsafe
+OPENDAL_UPYUN_ROOT=/path/to/dir
Review Comment:
typos.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437023460
##########
core/src/services/chainsafe/lister.rs:
##########
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::{parse_info, ChainsafeCore, Info};
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct ChainsafeLister {
+ core: Arc<ChainsafeCore>,
+ path: String,
+}
+
+impl ChainsafeLister {
+ pub fn new(core: Arc<ChainsafeCore>, path: &str) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ }
+ }
+}
+
+#[async_trait]
+impl oio::PageList for ChainsafeLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let resp = self.core.list_objects(&self.path).await?;
+
+ match resp.status() {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: Vec<Info> =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ for info in output {
+ let mut path = if self.path.starts_with('/') {
Review Comment:
I replace it:
```rust
// let mut path = if self.path.starts_with('/') {
// build_abs_path(&self.path, &info.name)
// } else {
// build_abs_path(&format!("/{}", &self.path), &info.name)
// };
let mut path = build_abs_path(&self.path, &info.name);
```
has this error:
```shell
---- behavior::test_list_with_recursive ----
test panicked: root must start with /
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1436983045
##########
core/src/services/chainsafe/lister.rs:
##########
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::{parse_info, ChainsafeCore, Info};
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct ChainsafeLister {
+ core: Arc<ChainsafeCore>,
+ path: String,
+}
+
+impl ChainsafeLister {
+ pub fn new(core: Arc<ChainsafeCore>, path: &str) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ }
+ }
+}
+
+#[async_trait]
+impl oio::PageList for ChainsafeLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let resp = self.core.list_objects(&self.path).await?;
+
+ match resp.status() {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: Vec<Info> =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ for info in output {
+ let mut path = if self.path.starts_with('/') {
Review Comment:
Because build_abs_path need root be start with /
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437023460
##########
core/src/services/chainsafe/lister.rs:
##########
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::{parse_info, ChainsafeCore, Info};
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct ChainsafeLister {
+ core: Arc<ChainsafeCore>,
+ path: String,
+}
+
+impl ChainsafeLister {
+ pub fn new(core: Arc<ChainsafeCore>, path: &str) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ }
+ }
+}
+
+#[async_trait]
+impl oio::PageList for ChainsafeLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let resp = self.core.list_objects(&self.path).await?;
+
+ match resp.status() {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: Vec<Info> =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ for info in output {
+ let mut path = if self.path.starts_with('/') {
Review Comment:
I replace it:
```rust
// let mut path = if self.path.starts_with('/') {
// build_abs_path(&self.path, &info.name)
// } else {
// build_abs_path(&format!("/{}", &self.path), &info.name)
// };
let mut path = build_abs_path(&self.path, &info.name);
```
has this error:
```shell
---- behavior::test_list_with_recursive ----
test panicked: root must start with /
```
Perhaps I should call 'normalize_root' during initialization?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437023460
##########
core/src/services/chainsafe/lister.rs:
##########
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::{parse_info, ChainsafeCore, Info};
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct ChainsafeLister {
+ core: Arc<ChainsafeCore>,
+ path: String,
+}
+
+impl ChainsafeLister {
+ pub fn new(core: Arc<ChainsafeCore>, path: &str) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ }
+ }
+}
+
+#[async_trait]
+impl oio::PageList for ChainsafeLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let resp = self.core.list_objects(&self.path).await?;
+
+ match resp.status() {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: Vec<Info> =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ for info in output {
+ let mut path = if self.path.starts_with('/') {
Review Comment:
I replace it:
```rust
// let mut path = if self.path.starts_with('/') {
// build_abs_path(&self.path, &info.name)
// } else {
// build_abs_path(&format!("/{}", &self.path), &info.name)
// };
let mut path = build_abs_path(&self.path, &info.name);
```
has this error:
```shell
---- behavior::test_list_with_recursive ----
test panicked: root must start with /
```
Perhaps I should call `normalize_root` during initialization?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] feat(services): add chainsafe support [incubator-opendal]
Posted by "hoslo (via GitHub)" <gi...@apache.org>.
hoslo commented on code in PR #3834:
URL: https://github.com/apache/incubator-opendal/pull/3834#discussion_r1437347194
##########
core/src/services/chainsafe/lister.rs:
##########
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::{parse_info, ChainsafeCore, Info};
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct ChainsafeLister {
+ core: Arc<ChainsafeCore>,
+ path: String,
+}
+
+impl ChainsafeLister {
+ pub fn new(core: Arc<ChainsafeCore>, path: &str) -> Self {
+ Self {
+ core,
+
+ path: path.to_string(),
+ }
+ }
+}
+
+#[async_trait]
+impl oio::PageList for ChainsafeLister {
+ async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+ let resp = self.core.list_objects(&self.path).await?;
+
+ match resp.status() {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+
+ let output: Vec<Info> =
+ serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+ for info in output {
+ let mut path = if self.path.starts_with('/') {
Review Comment:
Ok, this also fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@opendal.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org