You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/10/18 09:27:33 UTC
[incubator-opendal] branch main updated: refactor(ci/behavior_test): Migrate to 1password instead (#3338)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 06319530d refactor(ci/behavior_test): Migrate to 1password instead (#3338)
06319530d is described below
commit 06319530d6b68091b8ffa3da4578d11e65fbb8b2
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed Oct 18 17:27:26 2023 +0800
refactor(ci/behavior_test): Migrate to 1password instead (#3338)
* Save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Renaming
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Refactor python
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* refactor
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Refactor
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* remove debug log
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix file changed
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* compare with pr base
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Migrate
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* migrate existing s3 tests
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* fix secrets
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix typo
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix endpoint
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Test directly
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix plan
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* remove needs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Remove features
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---------
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
.../{test-core => behavior_test_core}/action.yaml | 9 +-
.../behavior_test/__init__.py} | 11 --
.../scripts/behavior_test/plan.py | 32 +++--
.../behavior_test/test_plan.py} | 37 +++--
.../services/fs/{local-fs => local_fs}/action.yml | 2 +-
.../services/{fs/local-fs => s3/aws_s3}/action.yml | 23 ++--
.github/services/s3/aws_s3_with_sse_c/action.yml | 43 ++++++
.../action.yml | 29 ++--
.../services/s3/{aws-s3 => minio_s3}/action.yml | 34 +++--
.../services/s3/minio_s3_with_anonymous/action.yml | 50 +++++++
.github/services/s3/{aws-s3 => r2}/action.yml | 32 +++--
.../{test_planner.yml => behavior_test.yml} | 14 +-
.../{core_test.yml => behavior_test_core.yml} | 54 ++------
.github/workflows/service_test_s3.yml | 150 +--------------------
.gitignore | 3 +
15 files changed, 234 insertions(+), 289 deletions(-)
diff --git a/.github/actions/test-core/action.yaml b/.github/actions/behavior_test_core/action.yaml
similarity index 87%
rename from .github/actions/test-core/action.yaml
rename to .github/actions/behavior_test_core/action.yaml
index 7e29b6b41..ca27b8d3f 100644
--- a/.github/actions/test-core/action.yaml
+++ b/.github/actions/behavior_test_core/action.yaml
@@ -22,9 +22,8 @@ inputs:
description: "The setup action for test"
service:
description: "The service to test"
- secrets:
- description: "The secrets for test"
- default: "{}"
+ feature:
+ description: "The feature to test"
runs:
using: "composite"
@@ -39,12 +38,10 @@ runs:
steps:
- name: Setup Test Core
uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }}
- with:
- secrets: ${{ toJson(inputs.secrets) }}
- name: Run Test Core
shell: bash
working-directory: core
- run: cargo nextest run behavior --archive-file=./core-test.tar.zst
+ run: cargo nextest run behavior --features ${{ inputs.feature }}
env:
OPENDAL_TEST: ${{ inputs.service }}
EOF
diff --git a/.github/services/fs/local-fs/action.yml b/.github/scripts/behavior_test/__init__.py
similarity index 78%
copy from .github/services/fs/local-fs/action.yml
copy to .github/scripts/behavior_test/__init__.py
index 72d54608c..13a83393a 100644
--- a/.github/services/fs/local-fs/action.yml
+++ b/.github/scripts/behavior_test/__init__.py
@@ -14,14 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-name: local_fs
-description: "Setup local fs"
-
-runs:
- using: "composite"
- steps:
- - name: Setup
- shell: bash
- run: |
- echo "OPENDAL_FS_ROOT=${{ runner.temp }}/" >> $GITHUB_ENV
diff --git a/scripts/workflow_planner.py b/.github/scripts/behavior_test/plan.py
similarity index 80%
rename from scripts/workflow_planner.py
rename to .github/scripts/behavior_test/plan.py
index 534c68ae5..9040d1a60 100755
--- a/scripts/workflow_planner.py
+++ b/.github/scripts/behavior_test/plan.py
@@ -21,9 +21,16 @@ import json
import os
from pathlib import Path
+# The path for current script.
+SCRIPT_PATH = Path(__file__).parent.absolute()
+# The path for `.github` dir.
+GITHUB_DIR = SCRIPT_PATH.parent.parent
+# The project dir for opendal.
+PROJECT_DIR = GITHUB_DIR.parent
+
def get_provided_cases():
- root_dir = ".github/services"
+ root_dir = f"{GITHUB_DIR}/services"
cases = [
{
@@ -62,7 +69,9 @@ def calculate_core_cases(cases, changed_files):
# If any of the core files changed, we will run all cases.
if any(
- p.startswith("core/src/") and not p.startswith("core/src/services")
+ p.startswith("core/")
+ and not p.startswith("core/src/services/")
+ and not p.endswith(".md")
for p in changed_files
):
return cases
@@ -89,11 +98,13 @@ def plan(changed_files):
jobs["core"] = [
{
"os": "ubuntu-latest",
- "features": ",".join(
- set([f"services-{v['service']}" for v in core_cases])
- ),
"cases": [
- {"setup": v["setup"], "service": v["service"]} for v in core_cases
+ {
+ "setup": v["setup"],
+ "service": v["service"],
+ "feature": "services-{}".format(v["service"].replace("_", "-")),
+ }
+ for v in core_cases
],
},
]
@@ -103,12 +114,13 @@ def plan(changed_files):
jobs["core"].append(
{
"os": "windows-latest",
- "features": "services-fs",
- "cases": [{"setup": "local-fs", "service": "fs"}],
+ "cases": [
+ {"setup": "local_fs", "service": "fs", "feature": "services-fs"}
+ ],
}
)
- return json.dumps(jobs)
+ return jobs
# For quick test:
@@ -117,4 +129,4 @@ def plan(changed_files):
if __name__ == "__main__":
changed_files = sys.argv[1:]
result = plan(changed_files)
- print(result)
+ print(json.dumps(result))
diff --git a/.github/services/fs/local-fs/action.yml b/.github/scripts/behavior_test/test_plan.py
similarity index 50%
copy from .github/services/fs/local-fs/action.yml
copy to .github/scripts/behavior_test/test_plan.py
index 72d54608c..090a56d14 100644
--- a/.github/services/fs/local-fs/action.yml
+++ b/.github/scripts/behavior_test/test_plan.py
@@ -15,13 +15,30 @@
# specific language governing permissions and limitations
# under the License.
-name: local_fs
-description: "Setup local fs"
-
-runs:
- using: "composite"
- steps:
- - name: Setup
- shell: bash
- run: |
- echo "OPENDAL_FS_ROOT=${{ runner.temp }}/" >> $GITHUB_ENV
+import unittest
+from plan import plan
+
+
+class BehaviorTestPlan(unittest.TestCase):
+ def test_empty(self):
+ result = plan([])
+ self.assertEqual(result, {})
+
+ def test_core_cargo_toml(self):
+ result = plan(["core/Cargo.toml"])
+ self.assertTrue(result["components"]["core"])
+
+ def test_core_services_fs(self):
+ result = plan(["core/src/services/fs/mod.rs"])
+ self.assertTrue(result["components"]["core"])
+ self.assertTrue(len(result["core"]) > 0)
+
+ cases = [v["service"] for v in result["core"][0]["cases"]]
+ # Should not contain fs
+ self.assertTrue("fs" in cases)
+ # Should not contain s3
+ self.assertFalse("s3" in cases)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/services/fs/local-fs/action.yml b/.github/services/fs/local_fs/action.yml
similarity index 95%
copy from .github/services/fs/local-fs/action.yml
copy to .github/services/fs/local_fs/action.yml
index 72d54608c..7b36dc3a9 100644
--- a/.github/services/fs/local-fs/action.yml
+++ b/.github/services/fs/local_fs/action.yml
@@ -16,7 +16,7 @@
# under the License.
name: local_fs
-description: "Setup local fs"
+description: 'Behavior test for local fs'
runs:
using: "composite"
diff --git a/.github/services/fs/local-fs/action.yml b/.github/services/s3/aws_s3/action.yml
similarity index 57%
rename from .github/services/fs/local-fs/action.yml
rename to .github/services/s3/aws_s3/action.yml
index 72d54608c..ce95ec863 100644
--- a/.github/services/fs/local-fs/action.yml
+++ b/.github/services/s3/aws_s3/action.yml
@@ -15,13 +15,20 @@
# specific language governing permissions and limitations
# under the License.
-name: local_fs
-description: "Setup local fs"
+name: aws_s3
+description: 'Behavior test for AWS S3. This service is sponsored by @datafuse_labs.'
runs:
- using: "composite"
- steps:
- - name: Setup
- shell: bash
- run: |
- echo "OPENDAL_FS_ROOT=${{ runner.temp }}/" >> $GITHUB_ENV
+ using: "composite"
+ steps:
+ - name: Setup
+ uses: 1password/load-secrets-action@v1
+ with:
+ export-env: true
+ env:
+ OPENDAL_S3_ROOT: op://services/s3/root
+ OPENDAL_S3_BUCKET: op://services/s3/bucket
+ OPENDAL_S3_ENDPOINT: op://services/s3/endpoint
+ OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id
+ OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key
+ OPENDAL_S3_REGION: op://services/s3/region
diff --git a/.github/services/s3/aws_s3_with_sse_c/action.yml b/.github/services/s3/aws_s3_with_sse_c/action.yml
new file mode 100644
index 000000000..321b290d6
--- /dev/null
+++ b/.github/services/s3/aws_s3_with_sse_c/action.yml
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: aws_s3_with_sse_c
+description: 'Behavior test for AWS S3 with SSE-C. This service is sponsored by @datafuse_labs.'
+
+runs:
+ using: "composite"
+ steps:
+ - name: Setup
+ uses: 1password/load-secrets-action@v1
+ with:
+ export-env: true
+ env:
+ OPENDAL_S3_ROOT: op://services/s3/root
+ OPENDAL_S3_BUCKET: op://services/s3/bucket
+ OPENDAL_S3_ENDPOINT: op://services/s3/endpoint
+ OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id
+ OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key
+ OPENDAL_S3_REGION: op://services/s3/region
+
+ - name: Add extra settings
+ shell: bash
+ run: |
+ cat << EOF >> $GITHUB_ENV
+ OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM=AES256
+ OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY=MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=
+ OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5=zZ5FnqcIqUjVwvWmyog4zw==
+ EOF
diff --git a/.github/services/s3/aws-s3/action.yml b/.github/services/s3/aws_s3_with_virtual_host/action.yml
similarity index 53%
copy from .github/services/s3/aws-s3/action.yml
copy to .github/services/s3/aws_s3_with_virtual_host/action.yml
index b560b4434..730068d1f 100644
--- a/.github/services/s3/aws-s3/action.yml
+++ b/.github/services/s3/aws_s3_with_virtual_host/action.yml
@@ -15,24 +15,25 @@
# specific language governing permissions and limitations
# under the License.
-name: aws_s3
-description: "Setup AWS S3"
+name: aws_s3_with_virtual_host
+description: 'Behavior test for AWS S3 with virtual host. This service is sponsored by @datafuse_labs.'
-inputs:
- secrets:
- description: "The secrets for test"
- default: "{}"
-
-# TODO: we should migrate to 1password instead.
runs:
using: "composite"
steps:
- name: Setup
+ uses: 1password/load-secrets-action@v1
+ with:
+ export-env: true
+ env:
+ OPENDAL_S3_ROOT: op://services/s3/root
+ OPENDAL_S3_BUCKET: op://services/s3/bucket
+ OPENDAL_S3_ENDPOINT: op://services/s3/endpoint
+ OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id
+ OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key
+ OPENDAL_S3_REGION: op://services/s3/region
+
+ - name: Add extra settings
shell: bash
run: |
- echo "OPENDAL_S3_ROOT=${{ fromJson(inputs.secrets).OPENDAL_S3_ROOT }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_BUCKET=${{ fromJson(inputs.secrets).OPENDAL_S3_BUCKET }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_ENDPOINT=${{ fromJson(inputs.secrets).OPENDAL_S3_ENDPOINT }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_ACCESS_KEY_ID=${{ fromJson(inputs.secrets).OPENDAL_S3_ACCESS_KEY_ID }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_SECRET_ACCESS_KEY=${{ fromJson(inputs.secrets).OPENDAL_S3_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_REGION=ap-northeast-1" >> $GITHUB_ENV
+ echo "OPENDAL_S3_ENABLE_VIRTUAL_HOST_STYLE=on" >> $GITHUB_ENV
diff --git a/.github/services/s3/aws-s3/action.yml b/.github/services/s3/minio_s3/action.yml
similarity index 53%
copy from .github/services/s3/aws-s3/action.yml
copy to .github/services/s3/minio_s3/action.yml
index b560b4434..ed237bb37 100644
--- a/.github/services/s3/aws-s3/action.yml
+++ b/.github/services/s3/minio_s3/action.yml
@@ -15,24 +15,30 @@
# specific language governing permissions and limitations
# under the License.
-name: aws_s3
-description: "Setup AWS S3"
+name: minio_s3
+description: 'Behavior test for Minio S3.'
-inputs:
- secrets:
- description: "The secrets for test"
- default: "{}"
-
-# TODO: we should migrate to 1password instead.
runs:
using: "composite"
steps:
+ - name: Setup MinIO Server
+ shell: bash
+ working-directory: fixtures/s3
+ run: docker-compose -f docker-compose-minio.yml up -d
+ - name: Setup test bucket
+ shell: bash
+ env:
+ AWS_ACCESS_KEY_ID: "minioadmin"
+ AWS_SECRET_ACCESS_KEY: "minioadmin"
+ AWS_EC2_METADATA_DISABLED: "true"
+ run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
- name: Setup
shell: bash
run: |
- echo "OPENDAL_S3_ROOT=${{ fromJson(inputs.secrets).OPENDAL_S3_ROOT }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_BUCKET=${{ fromJson(inputs.secrets).OPENDAL_S3_BUCKET }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_ENDPOINT=${{ fromJson(inputs.secrets).OPENDAL_S3_ENDPOINT }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_ACCESS_KEY_ID=${{ fromJson(inputs.secrets).OPENDAL_S3_ACCESS_KEY_ID }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_SECRET_ACCESS_KEY=${{ fromJson(inputs.secrets).OPENDAL_S3_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_REGION=ap-northeast-1" >> $GITHUB_ENV
+ cat << EOF >> $GITHUB_ENV
+ OPENDAL_S3_BUCKET=test
+ OPENDAL_S3_ENDPOINT=http://127.0.0.1:9000
+ OPENDAL_S3_ACCESS_KEY_ID=minioadmin
+ OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin
+ OPENDAL_S3_REGION=us-east-1
+ EOF
diff --git a/.github/services/s3/minio_s3_with_anonymous/action.yml b/.github/services/s3/minio_s3_with_anonymous/action.yml
new file mode 100644
index 000000000..407289d9c
--- /dev/null
+++ b/.github/services/s3/minio_s3_with_anonymous/action.yml
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: minio_s3_with_anonymous
+description: 'Behavior test for Minio S3 with anonymous access.'
+
+runs:
+ using: "composite"
+ steps:
+ - name: Setup MinIO Server
+ shell: bash
+ working-directory: fixtures/s3
+ run: |
+ docker-compose -f docker-compose-minio.yml up -d
+ - name: Setup test bucket
+ shell: bash
+ env:
+ AWS_ACCESS_KEY_ID: "minioadmin"
+ AWS_SECRET_ACCESS_KEY: "minioadmin"
+ AWS_EC2_METADATA_DISABLED: "true"
+ run: |
+ aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
+
+ curl -O https://dl.min.io/client/mc/release/linux-amd64/mc
+ chmod +x mc
+ ./mc alias set local http://127.0.0.1:9000/ minioadmin minioadmin
+ ./mc anonymous set public local/test
+ - name: Setup
+ shell: bash
+ run: |
+ cat << EOF >> $GITHUB_ENV
+ OPENDAL_S3_BUCKET=test
+ OPENDAL_S3_ENDPOINT=http://127.0.0.1:9000
+ OPENDAL_S3_REGION=us-east-1
+ OPENDAL_S3_ALLOW_ANONYMOUS=on
+ EOF
diff --git a/.github/services/s3/aws-s3/action.yml b/.github/services/s3/r2/action.yml
similarity index 53%
rename from .github/services/s3/aws-s3/action.yml
rename to .github/services/s3/r2/action.yml
index b560b4434..38b85141f 100644
--- a/.github/services/s3/aws-s3/action.yml
+++ b/.github/services/s3/r2/action.yml
@@ -15,24 +15,28 @@
# specific language governing permissions and limitations
# under the License.
-name: aws_s3
-description: "Setup AWS S3"
+name: r2
+description: 'Behavior test for Cloudflare R2. This service is sponsored by @Xuanwo.'
-inputs:
- secrets:
- description: "The secrets for test"
- default: "{}"
-
-# TODO: we should migrate to 1password instead.
runs:
using: "composite"
steps:
- name: Setup
+ uses: 1password/load-secrets-action@v1
+ with:
+ export-env: true
+ env:
+ OPENDAL_S3_BUCKET: op://services/r2/bucket
+ OPENDAL_S3_ENDPOINT: op://services/r2/endpoint
+ OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id
+ OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key
+
+ # OPENDAL_S3_BATCH_MAX_OPERATIONS is the R2's limitation
+ # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
+ - name: Add extra settings
shell: bash
run: |
- echo "OPENDAL_S3_ROOT=${{ fromJson(inputs.secrets).OPENDAL_S3_ROOT }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_BUCKET=${{ fromJson(inputs.secrets).OPENDAL_S3_BUCKET }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_ENDPOINT=${{ fromJson(inputs.secrets).OPENDAL_S3_ENDPOINT }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_ACCESS_KEY_ID=${{ fromJson(inputs.secrets).OPENDAL_S3_ACCESS_KEY_ID }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_SECRET_ACCESS_KEY=${{ fromJson(inputs.secrets).OPENDAL_S3_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV
- echo "OPENDAL_S3_REGION=ap-northeast-1" >> $GITHUB_ENV
+ cat << EOF >> $GITHUB_ENV
+ OPENDAL_S3_REGION=auto
+ OPENDAL_S3_BATCH_MAX_OPERATIONS=700
+ EOF
diff --git a/.github/workflows/test_planner.yml b/.github/workflows/behavior_test.yml
similarity index 88%
rename from .github/workflows/test_planner.yml
rename to .github/workflows/behavior_test.yml
index 29eabd6f9..a4b2b240d 100644
--- a/.github/workflows/test_planner.yml
+++ b/.github/workflows/behavior_test.yml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-name: Test
+name: Behavior Test
on:
push:
@@ -62,10 +62,9 @@ jobs:
has_secrets="true"
fi
- git fetch origin main:main
- files_changed=$(git diff --name-only main ${{ github.event.pull_request.head.sha }})
+ files_changed=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }})
echo "Files changed:"
- echo "$FILES_CHANGED"
+ echo "$files_changed"
fi
# Export variables
@@ -73,12 +72,12 @@ jobs:
export GITHUB_IS_PUSH=$is_push
# Run the workflow planner script
- PLAN=$(./scripts/workflow_planner.py $files_changed)
+ PLAN=$(./.github/scripts/behavior_test/plan.py $files_changed)
echo "Plan:"
echo "$PLAN" | jq .
echo "plan=$PLAN" >> $GITHUB_OUTPUT
- core_test:
+ test_core:
name: core / ${{ matrix.os }}
needs: [plan]
if: fromJson(needs.plan.outputs.plan).components.core
@@ -86,8 +85,7 @@ jobs:
strategy:
matrix:
include: ${{ fromJson(needs.plan.outputs.plan).core }}
- uses: ./.github/workflows/core_test.yml
+ uses: ./.github/workflows/behavior_test_core.yml
with:
os: ${{ matrix.os }}
cases: ${{ toJson(matrix.cases) }}
- features: ${{ matrix.features }}
diff --git a/.github/workflows/core_test.yml b/.github/workflows/behavior_test_core.yml
similarity index 58%
rename from .github/workflows/core_test.yml
rename to .github/workflows/behavior_test_core.yml
index 9cd213d4e..344fe62d7 100644
--- a/.github/workflows/core_test.yml
+++ b/.github/workflows/behavior_test_core.yml
@@ -26,47 +26,10 @@ on:
cases:
required: true
type: string
- features:
- required: true
- type: string
jobs:
- build:
- name: build
- runs-on: ${{ inputs.os }}
- steps:
- - uses: actions/checkout@v4
-
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- with:
- need-nextest: true
- need-protoc: true
- need-rocksdb: true
- github-token: ${{ secrets.GITHUB_TOKEN }}
-
- # Required by hdfs
- - name: Setup java env
- uses: actions/setup-java@v3
- with:
- distribution: temurin
- java-version: "11"
-
- - name: Build
- shell: bash
- working-directory: core
- run: |
- cargo nextest archive --features ${{ inputs.features }} --archive-file ./core-test.tar.zst
-
- - name: Upload
- uses: actions/upload-artifact@v3
- with:
- name: ${{ inputs.os }}-core-test
- path: ./core/core-test.tar.zst
-
test:
name: ${{ matrix.cases.service }} / ${{ matrix.cases.setup }}
- needs: [build]
runs-on: ${{ inputs.os }}
strategy:
matrix:
@@ -79,16 +42,19 @@ jobs:
need-nextest: true
need-protoc: true
need-rocksdb: true
+ github-token: ${{ secrets.GITHUB_TOKEN }}
- - name: Download
- uses: actions/download-artifact@v3
- with:
- name: ${{ inputs.os }}-core-test
- path: ./core
+ # TODO: 1password is only supported on linux
+ #
+ # Waiting for https://github.com/1Password/load-secrets-action/issues/46
+ - name: Setup service account token for 1password
+ if: runner.os == 'Linux'
+ shell: bash
+ run: echo "OP_SERVICE_ACCOUNT_TOKEN=${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" >> $GITHUB_ENV
- name: Test Core
- uses: ./.github/actions/test-core
+ uses: ./.github/actions/behavior_test_core
with:
setup: ${{ matrix.cases.setup }}
service: ${{ matrix.cases.service }}
- secrets: ${{ toJson(secrets) }}
+ feature: ${{ matrix.cases.feature }}
diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml
index 296ba03df..27d493bc7 100644
--- a/.github/workflows/service_test_s3.yml
+++ b/.github/workflows/service_test_s3.yml
@@ -39,54 +39,6 @@ concurrency:
cancel-in-progress: true
jobs:
- aws_s3_with_virtual_host:
- runs-on: ubuntu-latest
- if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- with:
- need-nextest: true
- - name: Test
- shell: bash
- working-directory: core
- run: cargo nextest run behavior
- env:
- OPENDAL_TEST: s3
- OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }}
- OPENDAL_S3_BUCKET: ${{ secrets.OPENDAL_S3_BUCKET }}
- OPENDAL_S3_ENDPOINT: ${{ secrets.OPENDAL_S3_ENDPOINT }}
- OPENDAL_S3_ACCESS_KEY_ID: ${{ secrets.OPENDAL_S3_ACCESS_KEY_ID }}
- OPENDAL_S3_SECRET_ACCESS_KEY: ${{ secrets.OPENDAL_S3_SECRET_ACCESS_KEY }}
- OPENDAL_S3_ENABLE_VIRTUAL_HOST_STYLE: on
- OPENDAL_S3_REGION: ap-northeast-1
-
- aws_s3_with_sse_c:
- runs-on: ubuntu-latest
- if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- with:
- need-nextest: true
- - name: Test
- shell: bash
- working-directory: core
- run: cargo nextest run behavior
- env:
- OPENDAL_TEST: s3
- OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }}
- OPENDAL_S3_BUCKET: ${{ secrets.OPENDAL_S3_BUCKET }}
- OPENDAL_S3_ENDPOINT: ${{ secrets.OPENDAL_S3_ENDPOINT }}
- OPENDAL_S3_ACCESS_KEY_ID: ${{ secrets.OPENDAL_S3_ACCESS_KEY_ID }}
- OPENDAL_S3_SECRET_ACCESS_KEY: ${{ secrets.OPENDAL_S3_SECRET_ACCESS_KEY }}
- OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: AES256
- OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY: MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=
- OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5: zZ5FnqcIqUjVwvWmyog4zw==
- OPENDAL_S3_REGION: ap-northeast-1
-
aws_s3_with_assume_role:
runs-on: ubuntu-latest
if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
@@ -119,106 +71,6 @@ jobs:
OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing
OPENDAL_S3_REGION: ap-northeast-1
- minio_s3:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
- - name: Setup MinIO Server
- shell: bash
- working-directory: fixtures/s3
- run: docker-compose -f docker-compose-minio.yml up -d
- - name: Setup test bucket
- env:
- AWS_ACCESS_KEY_ID: "minioadmin"
- AWS_SECRET_ACCESS_KEY: "minioadmin"
- AWS_EC2_METADATA_DISABLED: "true"
- run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- with:
- need-nextest: true
- - name: Test
- shell: bash
- working-directory: core
- run: cargo nextest run behavior
- env:
- OPENDAL_TEST: s3
- OPENDAL_S3_BUCKET: test
- OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000"
- OPENDAL_S3_ACCESS_KEY_ID: minioadmin
- OPENDAL_S3_SECRET_ACCESS_KEY: minioadmin
- OPENDAL_S3_REGION: us-east-1
-
- anonymous_minio_s3:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
- - name: Setup MinIO Server
- shell: bash
- working-directory: fixtures/s3
- run: |
- docker-compose -f docker-compose-minio.yml up -d
- - name: Setup test bucket
- env:
- AWS_ACCESS_KEY_ID: "minioadmin"
- AWS_SECRET_ACCESS_KEY: "minioadmin"
- AWS_EC2_METADATA_DISABLED: "true"
- run: |
- aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
-
- curl -O https://dl.min.io/client/mc/release/linux-amd64/mc
- chmod +x mc
- ./mc alias set local http://127.0.0.1:9000/ minioadmin minioadmin
- ./mc anonymous set public local/test
-
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- with:
- need-nextest: true
- - name: Test
- shell: bash
- working-directory: core
- run: cargo nextest run behavior
- env:
- OPENDAL_TEST: s3
- OPENDAL_S3_BUCKET: test
- OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000"
- OPENDAL_S3_ALLOW_ANONYMOUS: on
- OPENDAL_S3_REGION: us-east-1
-
- r2:
- runs-on: ubuntu-latest
- if: (github.event_name == 'push' && github.repository == 'apache/incubator-opendal') || !github.event.pull_request.head.repo.fork
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- with:
- need-nextest: true
-
- - name: Load secret
- id: op-load-secret
- uses: 1password/load-secrets-action@v1
- with:
- export-env: true
- env:
- OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
- OPENDAL_S3_BUCKET: op://services/r2/bucket
- OPENDAL_S3_ENDPOINT: op://services/r2/endpoint
- OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id
- OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key
-
- - name: Test
- shell: bash
- working-directory: core
- run: cargo nextest run behavior
- env:
- OPENDAL_TEST: s3
- OPENDAL_S3_REGION: auto
- # This is the R2's limitation
- # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
- OPENDAL_S3_BATCH_MAX_OPERATIONS: 700
-
java:
runs-on: ubuntu-latest
steps:
@@ -238,7 +90,7 @@ jobs:
- name: Test
shell: bash
working-directory: bindings/java
- run: ./mvnw test -Dtest="behavior.*Test"
+ run: ./mvnw test -Dtest="behavior.*Test"
env:
OPENDAL_TEST: s3
OPENDAL_S3_BUCKET: test
diff --git a/.gitignore b/.gitignore
index d5fd6da84..dc9a92312 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,6 @@ perf.*
# Yarn Integrity file
.yarn-integrity
+
+# Python cache files
+**/__pycache__/