You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/12/28 05:57:42 UTC
(incubator-opendal) branch main updated: refactor: Add edge test for aws assume role with web identity (#3839)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 90d8d516c refactor: Add edge test for aws assume role with web identity (#3839)
90d8d516c is described below
commit 90d8d516cb18c0f8369847bcd00c86c18b866b1e
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Thu Dec 28 13:57:34 2023 +0800
refactor: Add edge test for aws assume role with web identity (#3839)
* refactor: Add edge test for aws assume role with web identity
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Make clippy happy
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix b2
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---------
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
.github/workflows/edge_test.yml | 40 +++++++++++-
.github/workflows/service_test_s3.yml | 72 ----------------------
Cargo.lock | 9 +++
.../Cargo.toml | 29 +++++++++
.../s3_aws_assume_role_with_web_identity/README.md | 18 ++++++
.../src/main.rs | 33 ++++++++++
core/src/services/b2/backend.rs | 6 +-
7 files changed, 131 insertions(+), 76 deletions(-)
diff --git a/.github/workflows/edge_test.yml b/.github/workflows/edge_test.yml
index c15c3827a..a09f1de8b 100644
--- a/.github/workflows/edge_test.yml
+++ b/.github/workflows/edge_test.yml
@@ -29,6 +29,7 @@ on:
- "!core/src/docs/**"
- "!core/src/services/**"
- "core/src/services/fs/**"
+ - "core/src/services/s3/**"
- ".github/workflows/edge_test.yml"
jobs:
@@ -80,7 +81,7 @@ jobs:
unzip chrome-linux64.zip
cp -r chrome-linux64/ /tmp/chrome/
cp -r chromedriver-linux64 /tmp/chrome/chromedriver
-
+
- name: Setup wasm-pack
run: |
cargo install wasm-pack
@@ -97,9 +98,42 @@ jobs:
AWS_SECRET_ACCESS_KEY: "minioadmin"
AWS_EC2_METADATA_DISABLED: "true"
run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
-
+
- name: Test wasm
working-directory: core/edge/s3_read_on_wasm
run: |
export PATH=$PATH:/tmp/chrome/chrome-linux64/:/tmp/chrome/chromedriver-linux64/
- wasm-pack test --chrome --headless
\ No newline at end of file
+ wasm-pack test --chrome --headless
+
+ test_s3_aws_assume_role_with_web_identity:
+ runs-on: ubuntu-latest
+ if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
+ permissions:
+ id-token: write
+ contents: read
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup
+ - uses: actions/github-script@v7
+ id: id-token
+ with:
+ script: return await core.getIDToken("sts.amazonaws.com")
+ result-encoding: string
+ - name: Write ID token to file
+ working-directory: core/edge/s3_aws_assume_role_with_web_identity
+ run: echo "${{ steps.id-token.outputs.result }}" > web_identity_token
+
+ - name: Test
+ working-directory: core/edge/s3_aws_assume_role_with_web_identity
+ run: cargo run
+ env:
+ AWS_WEB_IDENTITY_TOKEN_FILE: web_identity_token
+ AWS_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing-assume
+ OPENDAL_TEST: s3
+ OPENDAL_S3_ROOT: CI/
+ OPENDAL_S3_BUCKET: opendal-testing
+ OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing
+ OPENDAL_S3_REGION: ap-northeast-1
diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml
deleted file mode 100644
index 6e585d475..000000000
--- a/.github/workflows/service_test_s3.yml
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Service Test S3
-
-on:
- push:
- branches:
- - main
- pull_request:
- branches:
- - main
- paths:
- - "core/src/**"
- - "core/tests/**"
- - "!core/src/docs/**"
- - "!core/src/services/**"
- - "core/src/services/s3/**"
- - "bindings/java/**"
- - ".github/workflows/service_test_s3.yml"
- - "fixtures/s3/**"
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
- cancel-in-progress: true
-
-jobs:
- aws_s3_with_assume_role:
- runs-on: ubuntu-latest
- if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
- permissions:
- id-token: write
- contents: read
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- with:
- need-nextest: true
- - uses: actions/github-script@v7
- id: id-token
- with:
- script: return await core.getIDToken("sts.amazonaws.com")
- result-encoding: string
- - name: Write ID token to file
- run: echo "${{ steps.id-token.outputs.result }}" > core/tests/data/web_identity_token
- - name: Test
- shell: bash
- working-directory: core
- run: cargo nextest run behavior --features tests
- env:
- AWS_WEB_IDENTITY_TOKEN_FILE: tests/data/web_identity_token
- AWS_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing-assume
- OPENDAL_TEST: s3
- OPENDAL_S3_ROOT: CI/
- OPENDAL_S3_BUCKET: opendal-testing
- OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing
- OPENDAL_S3_REGION: ap-northeast-1
diff --git a/Cargo.lock b/Cargo.lock
index de7e426d3..c5f52dc3b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2278,6 +2278,15 @@ dependencies = [
"signature 1.6.4",
]
+[[package]]
+name = "edge_test_aws_s3_assume_role_with_web_identity"
+version = "0.0.0"
+dependencies = [
+ "opendal",
+ "tokio",
+ "uuid",
+]
+
[[package]]
name = "edge_test_file_write_on_full_disk"
version = "0.0.0"
diff --git a/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml b/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml
new file mode 100644
index 000000000..e7cf869a7
--- /dev/null
+++ b/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+edition = "2021"
+name = "edge_test_aws_s3_assume_role_with_web_identity"
+publish = false
+version = "0.0.0"
+
+license.workspace = true
+
+[dependencies]
+opendal = { workspace = true, features = ["tests"] }
+uuid = { version = "1", features = ["serde", "v4"] }
+tokio = { version = "1", features = ["full"] }
diff --git a/core/edge/s3_aws_assume_role_with_web_identity/README.md b/core/edge/s3_aws_assume_role_with_web_identity/README.md
new file mode 100644
index 000000000..7fa237d8c
--- /dev/null
+++ b/core/edge/s3_aws_assume_role_with_web_identity/README.md
@@ -0,0 +1,18 @@
+# S3 AWS Assume Role With Web Identity
+
+This edge test case is for AWS s3 services that authed by `Assume Role With Web Identity`.
+
+For setup, please configure bucket and OIDC correctly, for example:
+
+```yaml
+- uses: actions/github-script@v7
+ id: id-token
+ with:
+ script: return await core.getIDToken("sts.amazonaws.com")
+ result-encoding: string
+- name: Write ID token to file
+ run: echo "${{ steps.id-token.outputs.result }}" > core/tests/data/web_identity_token
+```
+
+And configure `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN`.
+
diff --git a/core/edge/s3_aws_assume_role_with_web_identity/src/main.rs b/core/edge/s3_aws_assume_role_with_web_identity/src/main.rs
new file mode 100644
index 000000000..19446d2d7
--- /dev/null
+++ b/core/edge/s3_aws_assume_role_with_web_identity/src/main.rs
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use opendal::raw::tests::init_test_service;
+use opendal::{Result, Scheme};
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let op = init_test_service()?.expect("service must be init");
+ assert_eq!(op.info().scheme(), Scheme::S3);
+
+ let result = op
+ .is_exist(&uuid::Uuid::new_v4().to_string())
+ .await
+ .expect("this operation should never return error");
+ assert!(!result, "the file must be not exist");
+
+ Ok(())
+}
diff --git a/core/src/services/b2/backend.rs b/core/src/services/b2/backend.rs
index 7b910cb5b..1502afe0a 100644
--- a/core/src/services/b2/backend.rs
+++ b/core/src/services/b2/backend.rs
@@ -301,7 +301,11 @@ impl Accessor for B2Backend {
// The max multipart size of b2 is 5 Gb.
//
// ref: <https://www.backblaze.com/docs/cloud-storage-large-files>
- write_multi_max_size: Some(5 * 1024 * 1024 * 1024),
+ write_multi_max_size: if cfg!(target_pointer_width = "64") {
+ Some(5 * 1024 * 1024 * 1024)
+ } else {
+ Some(usize::MAX)
+ },
delete: true,
copy: true,