You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/10/29 06:10:21 UTC
(incubator-opendal) branch main updated: ci(services/azfile): add azfile integration test (#3409)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new b2b615b16 ci(services/azfile): add azfile integration test (#3409)
b2b615b16 is described below
commit b2b615b16f977dc3f889bc7bfe13a54106aea9b9
Author: Liuqing Yue <dq...@gmail.com>
AuthorDate: Sun Oct 29 14:10:17 2023 +0800
ci(services/azfile): add azfile integration test (#3409)
---
.github/services/azfile/azfile/action.yml | 39 +++++++++++++++++++++++++++++++
bindings/java/Cargo.toml | 2 ++
core/src/services/azfile/backend.rs | 8 ++++++-
core/src/services/azfile/core.rs | 37 +++++++++++++++++++++++++----
core/src/types/operator/builder.rs | 2 ++
core/src/types/scheme.rs | 2 ++
6 files changed, 84 insertions(+), 6 deletions(-)
diff --git a/.github/services/azfile/azfile/action.yml b/.github/services/azfile/azfile/action.yml
new file mode 100644
index 000000000..d2dc23cad
--- /dev/null
+++ b/.github/services/azfile/azfile/action.yml
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: azfile
+description: 'Behavior test for Azfile'
+
+runs:
+ using: "composite"
+ steps:
+ - name: Setup
+ uses: 1password/load-secrets-action@v1
+ with:
+ export-env: true
+ env:
+ OPENDAL_AZFILE_ACCOUNT_NAME: op://services/azfile/account_name
+ OPENDAL_AZFILE_ENDPOINT: op://services/azfile/endpoint
+ OPENDAL_AZFILE_ACCOUNT_KEY: op://services/azfile/account_key
+ OPENDAL_AZFILE_SHARE_NAME: op://services/azfile/share_name
+
+ - name: Add extra settings
+ shell: bash
+ run: |
+ cat << EOF >> $GITHUB_ENV
+ OPENDAL_AZFILE_ROOT=test/
+ EOF
diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml
index df1ca7e66..c5e7707b9 100644
--- a/bindings/java/Cargo.toml
+++ b/bindings/java/Cargo.toml
@@ -82,6 +82,7 @@ services-all = [
"services-wasabi",
"services-mongodb",
"services-sqlite",
+ "services-azfile",
]
# Default services provided by opendal.
@@ -129,6 +130,7 @@ services-wasabi = ["opendal/services-wasabi"]
services-mysql = ["opendal/services-mysql"]
services-mongodb = ["opendal/services-mongodb"]
services-sqlite = ["opendal/services-sqlite"]
+services-azfile = ["opendal/services-azfile"]
[dependencies]
anyhow = "1.0.71"
diff --git a/core/src/services/azfile/backend.rs b/core/src/services/azfile/backend.rs
index c7d8ad41c..75e475699 100644
--- a/core/src/services/azfile/backend.rs
+++ b/core/src/services/azfile/backend.rs
@@ -195,6 +195,7 @@ impl Builder for AzfileBuilder {
let config_loader = AzureStorageConfig {
account_name: Some(account_name),
+ account_key: self.account_key.clone(),
sas_token: self.sas_token.clone(),
..Default::default()
};
@@ -289,6 +290,11 @@ impl Accessor for AzfileBackend {
Ok(RpCreateDir::default())
}
_ => {
+ // we cannot just check status code because 409 Conflict has two meaning:
+ // 1. If a directory by the same name is being deleted when Create Directory is called, the server returns status code 409 (Conflict)
+ // 2. If a directory or file with the same name already exists, the operation fails with status code 409 (Conflict).
+ // but we just need case 2 (already exists)
+ // ref: https://learn.microsoft.com/en-us/rest/api/storageservices/create-directory
if resp
.headers()
.get("x-ms-error-code")
@@ -380,7 +386,7 @@ impl Accessor for AzfileBackend {
let status = resp.status();
match status {
- StatusCode::ACCEPTED => {
+ StatusCode::ACCEPTED | StatusCode::NOT_FOUND => {
resp.into_body().consume().await?;
Ok(RpDelete::default())
}
diff --git a/core/src/services/azfile/core.rs b/core/src/services/azfile/core.rs
index ce9717ac5..84fdc8a9f 100644
--- a/core/src/services/azfile/core.rs
+++ b/core/src/services/azfile/core.rs
@@ -38,9 +38,10 @@ use crate::*;
const X_MS_VERSION: &str = "x-ms-version";
const X_MS_WRITE: &str = "x-ms-write";
-const X_MS_RENAME_SOURCE: &str = "x-ms-rename-source";
+const X_MS_FILE_RENAME_SOURCE: &str = "x-ms-file-rename-source";
const X_MS_CONTENT_LENGTH: &str = "x-ms-content-length";
const X_MS_TYPE: &str = "x-ms-type";
+const X_MS_FILE_RENAME_REPLACE_IF_EXISTS: &str = "x-ms-file-rename-replace-if-exists";
pub struct AzfileCore {
pub root: String,
@@ -111,7 +112,19 @@ impl AzfileCore {
let mut req = Request::get(&url);
- req = req.header(RANGE, range.to_header());
+ if !range.is_full() {
+ // azfile doesn't support read with suffix range.
+ //
+ // ref: https://learn.microsoft.com/en-us/rest/api/storageservices/specifying-the-range-header-for-file-service-operations
+ if range.offset().is_none() && range.size().is_some() {
+ return Err(Error::new(
+ ErrorKind::Unsupported,
+ "azblob doesn't support read with suffix range",
+ ));
+ }
+
+ req = req.header(RANGE, range.to_header());
+ }
let mut req = req
.body(AsyncBody::Empty)
@@ -262,14 +275,14 @@ impl AzfileCore {
"{}/{}/{}?restype=directory&comp=rename",
self.endpoint,
self.share_name,
- percent_encode_path(&p)
+ percent_encode_path(&new_p)
)
} else {
format!(
"{}/{}/{}?comp=rename",
self.endpoint,
self.share_name,
- percent_encode_path(&p)
+ percent_encode_path(&new_p)
)
};
@@ -277,7 +290,21 @@ impl AzfileCore {
req = req.header(CONTENT_LENGTH, 0);
- req = req.header(X_MS_RENAME_SOURCE, percent_encode_path(&new_p));
+ // x-ms-file-rename-source specifies the file or directory to be renamed.
+ // the value must be a URL style path
+ // the official document does not mention the URL style path
+ // find the solution from the community FAQ and implementation of the Java-SDK
+ // ref: https://learn.microsoft.com/en-us/answers/questions/799611/azure-file-service-rest-api(rename)?page=1
+ let source_url = format!(
+ "{}/{}/{}",
+ self.endpoint,
+ self.share_name,
+ percent_encode_path(&p)
+ );
+
+ req = req.header(X_MS_FILE_RENAME_SOURCE, &source_url);
+
+ req = req.header(X_MS_FILE_RENAME_REPLACE_IF_EXISTS, "true");
let mut req = req
.body(AsyncBody::Empty)
diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs
index c7976e2c7..211444064 100644
--- a/core/src/types/operator/builder.rs
+++ b/core/src/types/operator/builder.rs
@@ -159,6 +159,8 @@ impl Operator {
Scheme::Azblob => Self::from_map::<services::Azblob>(map)?.finish(),
#[cfg(feature = "services-azdls")]
Scheme::Azdls => Self::from_map::<services::Azdls>(map)?.finish(),
+ #[cfg(feature = "services-azfile")]
+ Scheme::Azfile => Self::from_map::<services::Azfile>(map)?.finish(),
#[cfg(feature = "services-cacache")]
Scheme::Cacache => Self::from_map::<services::Cacache>(map)?.finish(),
#[cfg(feature = "services-cos")]
diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs
index 879dc6b9e..32b84c04c 100644
--- a/core/src/types/scheme.rs
+++ b/core/src/types/scheme.rs
@@ -165,6 +165,8 @@ impl Scheme {
Scheme::Azblob,
#[cfg(feature = "services-azdls")]
Scheme::Azdls,
+ #[cfg(feature = "services-azfile")]
+ Scheme::Azfile,
#[cfg(feature = "services-cacache")]
Scheme::Cacache,
#[cfg(feature = "services-cos")]