You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by ho...@apache.org on 2024/02/05 04:23:26 UTC
(opendal) 01/01: fix(services/chainsafe): fix 423 http status
This is an automated email from the ASF dual-hosted git repository.
hoslo pushed a commit to branch fix-chainsafe-423
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit ff25333b9718b8b9beba2d5143f1760bf174a47a
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Feb 5 11:35:12 2024 +0800
fix(services/chainsafe): fix 423 http status
---
.github/scripts/test_behavior/plan.py | 4 +++
.github/services/hdfs/hdfs_default/action.yml | 46 +++++++++++++++++++++++++++
.github/workflows/service_test_hdfs.yml | 36 ---------------------
core/src/services/chainsafe/error.rs | 4 +++
4 files changed, 54 insertions(+), 36 deletions(-)
diff --git a/.github/scripts/test_behavior/plan.py b/.github/scripts/test_behavior/plan.py
index 66a44b13e7..164d50c77b 100755
--- a/.github/scripts/test_behavior/plan.py
+++ b/.github/scripts/test_behavior/plan.py
@@ -198,6 +198,10 @@ def generate_language_binding_cases(
) -> list[dict[str, str]]:
cases = unique_cases(cases)
+ # Remove hdfs cases for java.
+ if language == "java":
+ cases = [v for v in cases if v["service"] != "hdfs"]
+
if os.getenv("GITHUB_IS_PUSH") == "true":
return cases
diff --git a/.github/services/hdfs/hdfs_default/action.yml b/.github/services/hdfs/hdfs_default/action.yml
new file mode 100644
index 0000000000..d4bdce0765
--- /dev/null
+++ b/.github/services/hdfs/hdfs_default/action.yml
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: hdfs_default
+description: 'Behavior test for hdfs default'
+
+runs:
+ using: "composite"
+ steps:
+ - name: Setup java env
+ uses: actions/setup-java@v4
+ with:
+ distribution: temurin
+ java-version: "11"
+ - name: Setup
+ shell: bash
+ run: |
+ curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
+
+ export HADOOP_HOME="/home/runner/hadoop-3.3.5"
+ export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
+
+ cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+
+ cat << EOF >> $GITHUB_ENV
+ HADOOP_HOME=${HADOOP_HOME}
+ CLASSPATH=${CLASSPATH}
+ LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
+ OPENDAL_HDFS_ROOT=/tmp/opendal/
+ OPENDAL_HDFS_NAME_NODE=default
+ OPENDAL_HDFS_ENABLE_APPEND=false
+ EOF
diff --git a/.github/workflows/service_test_hdfs.yml b/.github/workflows/service_test_hdfs.yml
index 1e1dae9f7e..c4447c2d45 100644
--- a/.github/workflows/service_test_hdfs.yml
+++ b/.github/workflows/service_test_hdfs.yml
@@ -37,42 +37,6 @@ concurrency:
cancel-in-progress: true
jobs:
- hdfs-default:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup
- with:
- need-nextest: true
-
- - name: Setup java env
- uses: actions/setup-java@v4
- with:
- distribution: temurin
- java-version: "11"
- - name: Setup hadoop env
- shell: bash
- run: |
- curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner
-
- - name: Test
- shell: bash
- working-directory: core
- run: |
- export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)
- export LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${{ env.HADOOP_HOME }}/lib/native
- cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${{ env.HADOOP_HOME }}/etc/hadoop/hdfs-site.xml
-
- cargo test behavior --features tests,services-hdfs
- env:
- HADOOP_HOME: "/home/runner/hadoop-3.3.5"
- OPENDAL_TEST: hdfs
- OPENDAL_HDFS_ROOT: /tmp/opendal/
- OPENDAL_HDFS_NAME_NODE: default
- OPENDAL_HDFS_ENABLE_APPEND: false
-
hdfs-cluster:
runs-on: ubuntu-latest
steps:
diff --git a/core/src/services/chainsafe/error.rs b/core/src/services/chainsafe/error.rs
index 5df3cebb3c..f998b625a4 100644
--- a/core/src/services/chainsafe/error.rs
+++ b/core/src/services/chainsafe/error.rs
@@ -46,6 +46,10 @@ pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
401 | 403 => (ErrorKind::PermissionDenied, false),
404 => (ErrorKind::NotFound, false),
304 | 412 => (ErrorKind::ConditionNotMatch, false),
+ // https://github.com/apache/opendal/issues/4146
+ // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/423
+ // We should retry it when we get 423 error.
+ 423 => (ErrorKind::RateLimited, true),
// Service like Upyun could return 499 error with a message like:
// Client Disconnect, we should retry it.
499 => (ErrorKind::Unexpected, true),