You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/10/18 12:03:06 UTC
[incubator-opendal] branch add-java-test-support created (now dacafce55)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a change to branch add-java-test-support
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
at dacafce55 Migrate redis
This branch includes the following new commits:
new 81dbf0bc5 Add java support
new dacafce55 Migrate redis
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-opendal] 01/02: Add java support
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch add-java-test-support
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 81dbf0bc56b1df0bdf1fbf9232a87a66a9c5bb7a
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed Oct 18 19:49:17 2023 +0800
Add java support
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
.../action.yaml | 15 +-
.github/actions/behavior_test_core/action.yaml | 1 -
.github/scripts/behavior_test/plan.py | 166 +++++++++++++++------
.github/scripts/behavior_test/test_plan.py | 12 +-
.github/workflows/behavior_test.yml | 13 ++
...est_core.yml => behavior_test_binding_java.yml} | 4 +-
.github/workflows/behavior_test_core.yml | 2 +-
7 files changed, 157 insertions(+), 56 deletions(-)
diff --git a/.github/actions/behavior_test_core/action.yaml b/.github/actions/behavior_test_binding_java/action.yaml
similarity index 78%
copy from .github/actions/behavior_test_core/action.yaml
copy to .github/actions/behavior_test_binding_java/action.yaml
index ca27b8d3f..635b109df 100644
--- a/.github/actions/behavior_test_core/action.yaml
+++ b/.github/actions/behavior_test_binding_java/action.yaml
@@ -31,20 +31,19 @@ runs:
- name: Setup
shell: bash
run: |
- mkdir -p ./dynamic_test_core &&
- cat <<EOF >./dynamic_test_core/action.yml
+ mkdir -p ./dynamic_test_binding_java &&
+ cat <<EOF >./dynamic_test_binding_java/action.yml
runs:
using: composite
steps:
- - name: Setup Test Core
+ - name: Setup Test
uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }}
- - name: Run Test Core
+ - name: Run Test Binding Java
shell: bash
- working-directory: core
- run: cargo nextest run behavior --features ${{ inputs.feature }}
+ working-directory: bindings/java
+ run: ./mvnw test -Dtest="behavior.*Test" -Dcargo-build.features=${{ inputs.feature }}
env:
OPENDAL_TEST: ${{ inputs.service }}
EOF
- name: Run
- id: run
- uses: ./dynamic_test_core
+ uses: ./dynamic_test_binding_java
diff --git a/.github/actions/behavior_test_core/action.yaml b/.github/actions/behavior_test_core/action.yaml
index ca27b8d3f..69674e0a7 100644
--- a/.github/actions/behavior_test_core/action.yaml
+++ b/.github/actions/behavior_test_core/action.yaml
@@ -46,5 +46,4 @@ runs:
OPENDAL_TEST: ${{ inputs.service }}
EOF
- name: Run
- id: run
uses: ./dynamic_test_core
diff --git a/.github/scripts/behavior_test/plan.py b/.github/scripts/behavior_test/plan.py
index 9040d1a60..1f378d352 100755
--- a/.github/scripts/behavior_test/plan.py
+++ b/.github/scripts/behavior_test/plan.py
@@ -19,7 +19,9 @@
import sys
import json
import os
+import re
from pathlib import Path
+from dataclasses import dataclass, field
# The path for current script.
SCRIPT_PATH = Path(__file__).parent.absolute()
@@ -29,13 +31,14 @@ GITHUB_DIR = SCRIPT_PATH.parent.parent
PROJECT_DIR = GITHUB_DIR.parent
-def get_provided_cases():
+def provided_cases():
root_dir = f"{GITHUB_DIR}/services"
cases = [
{
"service": service,
"setup": setup,
+ "feature": "services-{}".format(service.replace("_", "-")),
"content": Path(
os.path.join(root_dir, service, setup, "action.yml")
).read_text(),
@@ -50,64 +53,138 @@ def get_provided_cases():
if not os.getenv("GITHUB_HAS_SECRETS") == "true":
cases[:] = [v for v in cases if "secrets" not in v["content"]]
- return cases
+ # Remove content from cases.
+ return [
+ {
+ "setup": v["setup"],
+ "service": v["service"],
+ "feature": v["feature"],
+ }
+ for v in cases
+ ]
+
+
+@dataclass
+class Hint:
+ # Is core affected?
+ core: bool = field(default=False, init=False)
+ # Is binding java affected?
+ binding_java: bool = field(default=False, init=False)
+
+ # Should we run all services test?
+ all_service: bool = field(default=False, init=False)
+ # affected services set.
+ services: set = field(default_factory=set, init=False)
+
+
+def calculate_hint(changed_files):
+ hint = Hint()
+ # Remove all files that ends with `.md`
+ changed_files = [f for f in changed_files if not f.endswith(".md")]
-def calculate_core_cases(cases, changed_files):
- # If any of the core workflow changed, we will run all cases.
- for p in [
- ".github/workflows/core_test.yml",
- ".github/workflows/test_planner.yml",
- ".github/actions/test-core",
- ]:
- if p in changed_files:
- return cases
+ service_pattern = r"core/src/services/([^/]+)/"
+ test_pattern = r".github/services/[^/]+/"
+
+ for p in changed_files:
+ # core affected
+ if p.startswith("core/") and not p.startswith("core/src/services/"):
+ hint.core = True
+ hint.binding_java = True
+ hint.all_service = True
+
+ # binding java affected.
+ if p.startswith("bindings/java/"):
+ hint.binding_java = True
+ hint.all_service = True
+
+ # core service affected
+ match = re.search(service_pattern, p)
+ if match:
+ hint.core = True
+ hint.services.add(match.group(1))
+
+ # core test affected
+ match = re.search(test_pattern, p)
+ if match:
+ hint.core = True
+ hint.services.add(match.group(1))
+
+ return hint
+
+
+# unique_cases is used to only one setup for each service.
+#
+# We need this because we have multiple setup for each service and they have already been
+# tested by `core` workflow. So we can only test unique setup for each service for bindings.
+def unique_cases(cases):
+ ucases = {}
+ for case in cases:
+ service = case["service"]
+ if service not in ucases:
+ ucases[service] = case
+ # Convert the dictionary back to a list if needed
+ return list(ucases.values())
+
+
+def generate_core_cases(cases, hint):
# Always run all tests if it is a push event.
if os.getenv("GITHUB_IS_PUSH") == "true":
return cases
- # If any of the core files changed, we will run all cases.
- if any(
- p.startswith("core/")
- and not p.startswith("core/src/services/")
- and not p.endswith(".md")
- for p in changed_files
- ):
+ # Return empty if core is False
+ if not hint.core:
+ return []
+
+ # Return all services if all_service is True
+ if hint.all_service:
+ return cases
+
+ # Filter all cases that not shown un in changed files
+ cases = [v for v in cases if v["service"] in hint.services]
+ return cases
+
+
+def generate_binding_java_cases(cases, hint):
+ cases = unique_cases(cases)
+
+ # Always run all tests if it is a push event.
+ if os.getenv("GITHUB_IS_PUSH") == "true":
return cases
- if any(p.startswith("core/tests/") for p in changed_files):
+
+ # Return empty if core is False
+ if not hint.binding_java:
+ return []
+
+ # Return all services if all_service is True
+ if hint.all_service:
return cases
# Filter all cases that not shown un in changed files
- cases = [v for v in cases if any(v["service"] in p for p in changed_files)]
+ cases = [v for v in cases if v["service"] in hint.services]
return cases
-# Context is the github context: https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
def plan(changed_files):
- # TODO: add bindings/java, bindings/python in the future.
- components = ["core"]
- cases = get_provided_cases()
+ cases = provided_cases()
+ hint = calculate_hint(changed_files)
- core_cases = calculate_core_cases(cases, changed_files)
+ core_cases = generate_core_cases(cases, hint)
+ binding_java_cases = generate_binding_java_cases(cases, hint)
- jobs = {}
+ jobs = {
+ "components": {
+ "core": False,
+ "binding_java": False,
+ },
+ "core": [],
+ "binding_java": [],
+ }
if len(core_cases) > 0:
- jobs["components"] = {"core": True}
- jobs["core"] = [
- {
- "os": "ubuntu-latest",
- "cases": [
- {
- "setup": v["setup"],
- "service": v["service"],
- "feature": "services-{}".format(v["service"].replace("_", "-")),
- }
- for v in core_cases
- ],
- },
- ]
+ jobs["components"]["core"] = True
+ jobs["core"].append({"os": "ubuntu-latest", "cases": core_cases})
# fs is the only services need to run upon windows, let's hard code it here.
if "fs" in [v["service"] for v in core_cases]:
@@ -120,12 +197,15 @@ def plan(changed_files):
}
)
+ if len(binding_java_cases) > 0:
+ jobs["components"]["binding_java"] = True
+ jobs["binding_java"].append(
+ {"os": "ubuntu-latest", "cases": binding_java_cases}
+ )
+
return jobs
-# For quick test:
-#
-# ./scripts/workflow_planner.py PATH
if __name__ == "__main__":
changed_files = sys.argv[1:]
result = plan(changed_files)
diff --git a/.github/scripts/behavior_test/test_plan.py b/.github/scripts/behavior_test/test_plan.py
index 090a56d14..021001e44 100644
--- a/.github/scripts/behavior_test/test_plan.py
+++ b/.github/scripts/behavior_test/test_plan.py
@@ -22,7 +22,10 @@ from plan import plan
class BehaviorTestPlan(unittest.TestCase):
def test_empty(self):
result = plan([])
- self.assertEqual(result, {})
+ self.assertEqual(result["components"]["core"], False)
+ self.assertEqual(result["components"]["binding_java"], False)
+ self.assertEqual(len(result["core"]), 0)
+ self.assertEqual(len(result["binding_java"]), 0)
def test_core_cargo_toml(self):
result = plan(["core/Cargo.toml"])
@@ -39,6 +42,13 @@ class BehaviorTestPlan(unittest.TestCase):
# Should not contain s3
self.assertFalse("s3" in cases)
+ def test_binding_java(self):
+ result = plan(["bindings/java/pom.xml"])
+ self.assertFalse(result["components"]["core"])
+ self.assertTrue(len(result["core"]) == 0)
+ self.assertTrue(result["components"]["binding_java"])
+ self.assertTrue(len(result["binding_java"]) > 0)
+
if __name__ == "__main__":
unittest.main()
diff --git a/.github/workflows/behavior_test.yml b/.github/workflows/behavior_test.yml
index a4b2b240d..364eba63b 100644
--- a/.github/workflows/behavior_test.yml
+++ b/.github/workflows/behavior_test.yml
@@ -89,3 +89,16 @@ jobs:
with:
os: ${{ matrix.os }}
cases: ${{ toJson(matrix.cases) }}
+
+ test_binding_java:
+ name: binding_java / ${{ matrix.os }}
+ needs: [plan]
+ if: fromJson(needs.plan.outputs.plan).components.binding_java
+ secrets: inherit
+ strategy:
+ matrix:
+ include: ${{ fromJson(needs.plan.outputs.plan).binding_java }}
+ uses: ./.github/workflows/behavior_test_binding_java.yml
+ with:
+ os: ${{ matrix.os }}
+ cases: ${{ toJson(matrix.cases) }}
diff --git a/.github/workflows/behavior_test_core.yml b/.github/workflows/behavior_test_binding_java.yml
similarity index 95%
copy from .github/workflows/behavior_test_core.yml
copy to .github/workflows/behavior_test_binding_java.yml
index 344fe62d7..9eae4ce7c 100644
--- a/.github/workflows/behavior_test_core.yml
+++ b/.github/workflows/behavior_test_binding_java.yml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-name: Core
+name: Behavior Test Binding Java
on:
workflow_call:
@@ -53,7 +53,7 @@ jobs:
run: echo "OP_SERVICE_ACCOUNT_TOKEN=${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" >> $GITHUB_ENV
- name: Test Core
- uses: ./.github/actions/behavior_test_core
+ uses: ./.github/actions/behavior_test_binding_java
with:
setup: ${{ matrix.cases.setup }}
service: ${{ matrix.cases.service }}
diff --git a/.github/workflows/behavior_test_core.yml b/.github/workflows/behavior_test_core.yml
index 344fe62d7..2df44b82f 100644
--- a/.github/workflows/behavior_test_core.yml
+++ b/.github/workflows/behavior_test_core.yml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-name: Core
+name: Behavior Test Core
on:
workflow_call:
[incubator-opendal] 02/02: Migrate redis
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch add-java-test-support
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit dacafce55ab65a1488752aff015aff071593230e
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed Oct 18 20:02:56 2023 +0800
Migrate redis
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
.github/scripts/behavior_test/plan.py | 8 ++-
.github/services/redis/redis/action.yml | 40 +++++++++++++
.../services/redis/redis_with_cluster/action.yml | 35 +++++++++++
.github/workflows/service_test_redis.yml | 68 ----------------------
.github/workflows/service_test_s3.yml | 28 ---------
5 files changed, 82 insertions(+), 97 deletions(-)
diff --git a/.github/scripts/behavior_test/plan.py b/.github/scripts/behavior_test/plan.py
index 1f378d352..d24f4b020 100755
--- a/.github/scripts/behavior_test/plan.py
+++ b/.github/scripts/behavior_test/plan.py
@@ -54,7 +54,7 @@ def provided_cases():
cases[:] = [v for v in cases if "secrets" not in v["content"]]
# Remove content from cases.
- return [
+ cases = [
{
"setup": v["setup"],
"service": v["service"],
@@ -63,6 +63,10 @@ def provided_cases():
for v in cases
]
+ # Make sure the order is stable.
+ sorted_cases = sorted(cases, key=lambda x: (x["service"], x["setup"]))
+ return sorted_cases
+
@dataclass
class Hint:
@@ -117,6 +121,8 @@ def calculate_hint(changed_files):
#
# We need this because we have multiple setup for each service and they have already been
# tested by `core` workflow. So we can only test unique setup for each service for bindings.
+#
+# We make sure that we return the first setup for each service in alphabet order.
def unique_cases(cases):
ucases = {}
for case in cases:
diff --git a/.github/services/redis/redis/action.yml b/.github/services/redis/redis/action.yml
new file mode 100644
index 000000000..5fc68b8cb
--- /dev/null
+++ b/.github/services/redis/redis/action.yml
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: redis
+description: 'Behavior test for redis'
+
+runs:
+ using: "composite"
+ steps:
+ - name: Setup Redis with TLS
+ shell: bash
+ working-directory: fixtures/redis
+ run: |
+ # Install the CA in the system
+ sudo cp ssl/ca.crt /usr/local/share/ca-certificates
+ sudo update-ca-certificates
+
+ docker-compose -f docker-compose-redis-tls.yml up -d
+ - name: Setup
+ shell: bash
+ run: |
+ cat << EOF >> $GITHUB_ENV
+ OPENDAL_REDIS_ENDPOINT=rediss://localhost:6379
+ OPENDAL_REDIS_ROOT=/
+ OPENDAL_REDIS_DB=0
+ EOF
diff --git a/.github/services/redis/redis_with_cluster/action.yml b/.github/services/redis/redis_with_cluster/action.yml
new file mode 100644
index 000000000..4caeb85c3
--- /dev/null
+++ b/.github/services/redis/redis_with_cluster/action.yml
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: redis_with_cluster
+description: 'Behavior test for redis with cluster'
+
+runs:
+ using: "composite"
+ steps:
+ - name: Setup Redis Cluster
+ shell: bash
+ working-directory: fixtures/redis
+ run: docker-compose -f docker-compose-redis-cluster.yml up -d
+ - name: Setup
+ shell: bash
+ run: |
+ cat << EOF >> $GITHUB_ENV
+ OPENDAL_REDIS_CLUSTER_ENDPOINTS=redis://127.0.0.1:6380/,redis://127.0.0.1:6381/,redis://127.0.0.1:6382/,redis://127.0.0.1:6383/,redis://127.0.0.1:6384/,redis://127.0.0.1:6385/
+ OPENDAL_REDIS_ROOT=/test/opendal
+ OPENDAL_REDIS_DB=0
+ EOF
diff --git a/.github/workflows/service_test_redis.yml b/.github/workflows/service_test_redis.yml
index eae70466f..ef8cb682e 100644
--- a/.github/workflows/service_test_redis.yml
+++ b/.github/workflows/service_test_redis.yml
@@ -39,28 +39,6 @@ concurrency:
cancel-in-progress: true
jobs:
- redis:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
- - name: Setup Redis Server
- shell: bash
- working-directory: fixtures/redis
- run: docker-compose -f docker-compose-redis.yml up -d
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- with:
- need-nextest: true
- - name: Test
- shell: bash
- working-directory: core
- run: cargo nextest run behavior --features services-redis
- env:
- OPENDAL_TEST: redis
- OPENDAL_REDIS_ENDPOINT: tcp://127.0.0.1:6379
- OPENDAL_REDIS_ROOT: /
- OPENDAL_REDIS_DB: 0
-
redis-tls:
runs-on: ubuntu-latest
steps:
@@ -91,29 +69,6 @@ jobs:
OPENDAL_REDIS_ROOT: /
OPENDAL_REDIS_DB: 0
- redis-cluster:
- runs-on: ubuntu-latest
-
- steps:
- - uses: actions/checkout@v4
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- with:
- need-nextest: true
- - name: Setup Redis Cluster
- shell: bash
- working-directory: fixtures/redis
- run: docker-compose -f docker-compose-redis-cluster.yml up -d
- - name: Test
- shell: bash
- working-directory: core
- run: cargo nextest run behavior --features services-redis
- env:
- OPENDAL_TEST: redis
- OPENDAL_REDIS_CLUSTER_ENDPOINTS: redis://127.0.0.1:6380/,redis://127.0.0.1:6381/,redis://127.0.0.1:6382/,redis://127.0.0.1:6383/,redis://127.0.0.1:6384/,redis://127.0.0.1:6385/
- OPENDAL_REDIS_ROOT: /test/opendal
- OPENDAL_REDIS_DB: 0
-
redis-cluster-tls:
runs-on: ubuntu-latest
@@ -193,26 +148,3 @@ jobs:
OPENDAL_REDIS_ENDPOINT: tcp://127.0.0.1:6379
OPENDAL_REDIS_ROOT: /
OPENDAL_REDIS_DB: 0
-
- java:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
-
- - name: Setup Redis Server
- shell: bash
- working-directory: fixtures/redis
- run: docker-compose -f docker-compose-redis.yml up -d
-
- - name: Test
- shell: bash
- working-directory: bindings/java
- run: ./mvnw test -Dtest="behavior.*Test" -Dcargo-build.features=services-redis
- env:
- OPENDAL_TEST: redis
- OPENDAL_REDIS_ENDPOINT: tcp://127.0.0.1:6379
- OPENDAL_REDIS_ROOT: /
- OPENDAL_REDIS_DB: 0
diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml
index 27d493bc7..247080584 100644
--- a/.github/workflows/service_test_s3.yml
+++ b/.github/workflows/service_test_s3.yml
@@ -71,34 +71,6 @@ jobs:
OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing
OPENDAL_S3_REGION: ap-northeast-1
- java:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
- - name: Setup MinIO Server
- shell: bash
- working-directory: fixtures/s3
- run: docker-compose -f docker-compose-minio.yml up -d
- - name: Setup test bucket
- env:
- AWS_ACCESS_KEY_ID: "minioadmin"
- AWS_SECRET_ACCESS_KEY: "minioadmin"
- AWS_EC2_METADATA_DISABLED: "true"
- run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- - name: Test
- shell: bash
- working-directory: bindings/java
- run: ./mvnw test -Dtest="behavior.*Test"
- env:
- OPENDAL_TEST: s3
- OPENDAL_S3_BUCKET: test
- OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000"
- OPENDAL_S3_ACCESS_KEY_ID: minioadmin
- OPENDAL_S3_SECRET_ACCESS_KEY: minioadmin
- OPENDAL_S3_REGION: us-east-1
-
python:
runs-on: ubuntu-latest
steps: