You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by pi...@apache.org on 2022/03/08 13:31:18 UTC
[ozone] branch master updated: HDDS-6153. Add simple integration test to the read-replicas debug tool. (#3065)
This is an automated email from the ASF dual-hosted git repository.
pifta pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 58207a2 HDDS-6153. Add simple integration test to the read-replicas debug tool. (#3065)
58207a2 is described below
commit 58207a20df486f4cc8e4675e263eb42e3a5b7217
Author: Zita Dombi <50...@users.noreply.github.com>
AuthorDate: Tue Mar 8 14:31:03 2022 +0100
HDDS-6153. Add simple integration test to the read-replicas debug tool. (#3065)
---
.../dist/src/main/compose/ozone/docker-config | 4 ++
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 56 +++++++++++++++++++
.../debug/ozone-debug-corrupt-block.robot | 41 ++++++++++++++
.../debug/ozone-debug-dead-datanode.robot | 44 +++++++++++++++
.../debug/ozone-debug-stale-datanode.robot | 45 +++++++++++++++
.../main/smoketest/debug/ozone-debug-tests.robot | 58 ++++++++++++++++++++
.../src/main/smoketest/debug/ozone-debug.robot | 64 +++++++++++++++++-----
8 files changed, 299 insertions(+), 15 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
index 4b59e0e..2733e07 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
@@ -21,6 +21,7 @@ OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.http-address=scm:9876
OZONE-SITE.XML_ozone.scm.container.size=1GB
+OZONE-SITE.XML_ozone.scm.block.size=1MB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
@@ -38,6 +39,9 @@ OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_hdds.container.report.interval=60s
+OZONE-SITE.XML_ozone.scm.stale.node.interval=30s
+OZONE-SITE.XML_ozone.scm.dead.node.interval=45s
+OZONE-SITE.XML_hdds.heartbeat.interval=5s
OZONE_CONF_DIR=/etc/hadoop
OZONE_LOG_DIR=/var/log/hadoop
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
index 59a90ed..acc18a5 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
@@ -50,6 +50,8 @@ execute_robot_test scm freon
execute_robot_test scm cli
execute_robot_test scm admincli
+execute_debug_tests
+
execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-fso-ofs-link ozonefs/ozonefs.robot
execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-fso-o3fs-bucket ozonefs/ozonefs.robot
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 559f189..7af9040 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -422,3 +422,59 @@ prepare_for_runner_image() {
export OZONE_IMAGE="${runner_image}:${v}"
}
+## @description Executing the Ozone Debug CLI related robot tests
+execute_debug_tests() {
+
+ OZONE_DEBUG_VOLUME="cli-debug-volume"
+ OZONE_DEBUG_BUCKET="cli-debug-bucket"
+ OZONE_DEBUG_KEY="testfile"
+
+ execute_robot_test datanode debug/ozone-debug-tests.robot
+
+ corrupt_block_on_datanode
+ execute_robot_test datanode debug/ozone-debug-corrupt-block.robot
+
+ docker stop ozone_datanode_2
+
+ wait_for_datanode datanode_2 STALE 60
+ execute_robot_test datanode debug/ozone-debug-stale-datanode.robot
+ wait_for_datanode datanode_2 DEAD 60
+ execute_robot_test datanode debug/ozone-debug-dead-datanode.robot
+
+ docker start ozone_datanode_2
+
+ wait_for_datanode datanode_2 HEALTHY 60
+}
+
+## @description Corrupt a block on a datanode
+corrupt_block_on_datanode() {
+ docker-compose exec -T ${SCM} bash -c "ozone debug chunkinfo ${OZONE_DEBUG_VOLUME}/${OZONE_DEBUG_BUCKET}/${OZONE_DEBUG_KEY}" > /tmp/blocks
+ block=$(cat /tmp/blocks | jq -r '.KeyLocations[0][0].Locations.files[0]')
+ docker exec ozone_datanode_2 sed -i -e '1s/^/a/' "${block}"
+}
+
+## @description Wait for datanode state
+## @param Datanode name, eg datanode_1 datanode_2
+## @param State to check for
+## @param The maximum time to wait in seconds
+wait_for_datanode() {
+ local datanode=$1
+ local state=$2
+ local timeout=$3
+
+ SECONDS=0
+ while [[ $SECONDS -lt $timeout ]]; do
+ local command="ozone admin datanode list"
+ docker-compose exec -T ${SCM} bash -c "$command" | grep -A2 "$datanode" > /tmp/dn_check
+ local health=$(grep -c "State: $state" /tmp/dn_check)
+
+ if [[ "$health" -eq 1 ]]; then
+ echo "$datanode is $state"
+ return
+ else
+ echo "Waiting for $datanode to be $state"
+ fi
+ echo "SECONDS: $SECONDS"
+ done
+ echo "WARNING: $datanode is still not $state"
+}
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot
new file mode 100644
index 0000000..41e6b68
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test read-replicas in case of a corrupt replica
+Library OperatingSystem
+Resource ../lib/os.robot
+Resource ozone-debug.robot
+Test Timeout 5 minute
+*** Variables ***
+${VOLUME} cli-debug-volume
+${BUCKET} cli-debug-bucket
+${TESTFILE} testfile
+
+*** Test Cases ***
+Test ozone debug read-replicas with corrupt block replica
+ ${directory} = Execute read-replicas CLI tool
+ ${dn1_md5sum} = Execute cat ${directory}/${TESTFILE}_block1_ozone_datanode_1.ozone_default ${directory}/${TESTFILE}_block2_ozone_datanode_1.ozone_default | md5sum | awk '{print $1}'
+ ${dn2_md5sum} = Execute cat ${directory}/${TESTFILE}_block1_ozone_datanode_2.ozone_default ${directory}/${TESTFILE}_block2_ozone_datanode_2.ozone_default | md5sum | awk '{print $1}'
+ ${dn3_md5sum} = Execute cat ${directory}/${TESTFILE}_block1_ozone_datanode_3.ozone_default ${directory}/${TESTFILE}_block2_ozone_datanode_3.ozone_default | md5sum | awk '{print $1}'
+ ${testfile_md5sum} = Execute md5sum testfile | awk '{print $1}'
+ Should Be Equal ${dn1_md5sum} ${testfile_md5sum}
+ Should Not Be Equal ${dn2_md5sum} ${testfile_md5sum}
+ Should Be Equal ${dn3_md5sum} ${testfile_md5sum}
+ ${manifest} = Get File ${directory}/${TESTFILE}_manifest
+ ${json} = Evaluate json.loads('''${manifest}''') json
+ Compare JSON ${json}
+ Check for all datanodes ${json}
+ Check checksum mismatch error ${json} ozone_datanode_2.ozone_default
diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot
new file mode 100644
index 0000000..5df987a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test read-replicas in case of one datanode is unavailable
+Library OperatingSystem
+Resource ../lib/os.robot
+Resource ozone-debug.robot
+Test Timeout 5 minute
+*** Variables ***
+${VOLUME} cli-debug-volume
+${BUCKET} cli-debug-bucket
+${TESTFILE} testfile
+
+*** Test Cases ***
+Test ozone debug read-replicas with one datanode DEAD
+ ${directory} = Execute read-replicas CLI tool
+ ${count_files} = Count Files In Directory ${directory}
+ Should Be Equal As Integers ${count_files} 5
+ ${dn1_md5sum} = Execute cat ${directory}/${TESTFILE}_block1_ozone_datanode_1.ozone_default ${directory}/${TESTFILE}_block2_ozone_datanode_1.ozone_default | md5sum | awk '{print $1}'
+ ${dn3_md5sum} = Execute cat ${directory}/${TESTFILE}_block1_ozone_datanode_3.ozone_default ${directory}/${TESTFILE}_block2_ozone_datanode_3.ozone_default | md5sum | awk '{print $1}'
+ ${testfile_md5sum} = Execute md5sum testfile | awk '{print $1}'
+ Should Be Equal ${dn1_md5sum} ${testfile_md5sum}
+ Should Be Equal ${dn3_md5sum} ${testfile_md5sum}
+ ${manifest} = Get File ${directory}/${TESTFILE}_manifest
+ ${json} = Evaluate json.loads('''${manifest}''') json
+ Compare JSON ${json}
+ ${datanodes_expected} = Create List ozone_datanode_1.ozone_default ozone_datanode_3.ozone_default
+ ${datanodes_b1} = Create List ${json}[blocks][0][replicas][0][hostname] ${json}[blocks][0][replicas][1][hostname]
+ Check for datanodes ${datanodes_b1} ${datanodes_expected}
+ ${datanodes_b2} = Create List ${json}[blocks][1][replicas][0][hostname] ${json}[blocks][1][replicas][1][hostname]
+ Check for datanodes ${datanodes_b2} ${datanodes_expected}
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot
new file mode 100644
index 0000000..9225606
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test read-replicas in case of one datanode is unavailable
+Library OperatingSystem
+Resource ../lib/os.robot
+Resource ozone-debug.robot
+Test Timeout 5 minute
+*** Variables ***
+${VOLUME} cli-debug-volume
+${BUCKET} cli-debug-bucket
+${TESTFILE} testfile
+
+*** Test Cases ***
+Test ozone debug read-replicas with one datanode STALE
+ ${directory} = Execute read-replicas CLI tool
+ ${count_files} = Count Files In Directory ${directory}
+ Should Be Equal As Integers ${count_files} 7
+ ${corrupted_block1} = Get File Size ${directory}/${TESTFILE}_block1_ozone_datanode_2.ozone_default
+ ${corrupted_block2} = Get File Size ${directory}/${TESTFILE}_block2_ozone_datanode_2.ozone_default
+ Should Be Equal As Integers ${corrupted_block1} 0
+ Should Be Equal As Integers ${corrupted_block2} 0
+ ${dn1_md5sum} = Execute cat ${directory}/${TESTFILE}_block1_ozone_datanode_1.ozone_default ${directory}/${TESTFILE}_block2_ozone_datanode_1.ozone_default | md5sum | awk '{print $1}'
+ ${dn3_md5sum} = Execute cat ${directory}/${TESTFILE}_block1_ozone_datanode_3.ozone_default ${directory}/${TESTFILE}_block2_ozone_datanode_3.ozone_default | md5sum | awk '{print $1}'
+ ${testfile_md5sum} = Execute md5sum testfile | awk '{print $1}'
+ Should Be Equal ${dn1_md5sum} ${testfile_md5sum}
+ Should Be Equal ${dn3_md5sum} ${testfile_md5sum}
+ ${manifest} = Get File ${directory}/${TESTFILE}_manifest
+ ${json} = Evaluate json.loads('''${manifest}''') json
+ Compare JSON ${json}
+ Check for all datanodes ${json}
+ Check unavailable datanode error ${json} ozone_datanode_2.ozone_default
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
new file mode 100644
index 0000000..68cf29c
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test ozone Debug CLI
+Library OperatingSystem
+Resource ../lib/os.robot
+Resource ozone-debug.robot
+Test Timeout 5 minute
+Suite Setup Write keys
+*** Variables ***
+${VOLUME} cli-debug-volume
+${BUCKET} cli-debug-bucket
+${DEBUGKEY} debugKey
+${TESTFILE} testfile
+
+*** Keywords ***
+Write keys
+ Execute ozone sh volume create o3://om/${VOLUME} --space-quota 100TB --namespace-quota 100
+ Execute ozone sh bucket create o3://om/${VOLUME}/${BUCKET}
+ Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/${DEBUGKEY} /opt/hadoop/NOTICE.txt
+ Execute dd if=/dev/urandom of=testfile bs=100000 count=15
+ Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/${TESTFILE} testfile
+
+*** Test Cases ***
+Test ozone debug chunkinfo
+ ${result} = Execute ozone debug chunkinfo o3://om/${VOLUME}/${BUCKET}/${DEBUGKEY} | jq -r '.KeyLocations[0][0].Locations'
+ Should contain ${result} files
+ ${result} = Execute ozone debug chunkinfo o3://om/${VOLUME}/${BUCKET}/${DEBUGKEY} | jq -r '.KeyLocations[0][0].Locations.files[0]'
+ File Should Exist ${result}
+
+Test ozone debug read-replicas
+ ${directory} = Execute read-replicas CLI tool
+ ${count_files} = Count Files In Directory ${directory}
+ Should Be Equal As Integers ${count_files} 7
+ ${dn1_md5sum} = Execute cat ${directory}/${TESTFILE}_block1_ozone_datanode_1.ozone_default ${directory}/${TESTFILE}_block2_ozone_datanode_1.ozone_default | md5sum | awk '{print $1}'
+ ${dn2_md5sum} = Execute cat ${directory}/${TESTFILE}_block1_ozone_datanode_2.ozone_default ${directory}/${TESTFILE}_block2_ozone_datanode_2.ozone_default | md5sum | awk '{print $1}'
+ ${dn3_md5sum} = Execute cat ${directory}/${TESTFILE}_block1_ozone_datanode_3.ozone_default ${directory}/${TESTFILE}_block2_ozone_datanode_3.ozone_default | md5sum | awk '{print $1}'
+ ${testfile_md5sum} = Execute md5sum testfile | awk '{print $1}'
+ Should Be Equal ${dn1_md5sum} ${testfile_md5sum}
+ Should Be Equal ${dn2_md5sum} ${testfile_md5sum}
+ Should Be Equal ${dn3_md5sum} ${testfile_md5sum}
+ ${manifest} = Get File ${directory}/${TESTFILE}_manifest
+ ${json} = Evaluate json.loads('''${manifest}''') json
+ Compare JSON ${json}
+ Check for all datanodes ${json}
diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
index a70e2e7..7d6c211 100644
--- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
@@ -15,22 +15,56 @@
*** Settings ***
Documentation Test ozone Debug CLI
-Library OperatingSystem
-Resource ../commonlib.robot
-Test Timeout 2 minute
-Suite Setup Write key
-*** Variables ***
+Library Collections
+Resource ../lib/os.robot
*** Keywords ***
-Write key
- Execute ozone sh volume create o3://om/vol1 --space-quota 100TB --namespace-quota 100
- Execute ozone sh bucket create o3://om/vol1/bucket1
- Execute ozone sh key put o3://om/vol1/bucket1/debugKey /opt/hadoop/NOTICE.txt
+Execute read-replicas CLI tool
+ Execute ozone debug read-replicas o3://om/${VOLUME}/${BUCKET}/${TESTFILE}
+ ${directory} = Execute ls -d /opt/hadoop/${VOLUME}_${BUCKET}_${TESTFILE}_*/ | tail -n 1
+ Directory Should Exist ${directory}
+ File Should Exist ${directory}/${TESTFILE}_manifest
+ [Return] ${directory}
-*** Test Cases ***
-Test ozone debug
- ${result} = Execute ozone debug chunkinfo o3://om/vol1/bucket1/debugKey | jq -r '.KeyLocations[0][0].Locations'
- Should contain ${result} files
- ${result} = Execute ozone debug chunkinfo o3://om/vol1/bucket1/debugKey | jq -r '.KeyLocations[0][0].Locations.files[0]'
- File Should Exist ${result}
+Compare JSON
+ [arguments] ${json}
+ Should Be Equal ${json}[filename] ${VOLUME}/${BUCKET}/${TESTFILE}
+ ${file_size} = Get File Size ${TESTFILE}
+ Should Be Equal ${json}[datasize] ${file_size}
+ Should Be Equal As Integers ${json}[blocks][0][blockIndex] 1
+ Should Not Be Empty Convert To String ${json}[blocks][0][containerId]
+ Should Not Be Empty Convert To String ${json}[blocks][0][localId]
+ Should Be Equal As Integers ${json}[blocks][0][length] 1048576
+ Should Not Be Empty Convert To String ${json}[blocks][0][offset]
+ Should Be Equal As Integers ${json}[blocks][1][blockIndex] 2
+ Should Not Be Empty Convert To String ${json}[blocks][1][containerId]
+ Should Not Be Empty Convert To String ${json}[blocks][1][localId]
+ Should Be Equal As Integers ${json}[blocks][1][length] 451424
+ Should Not Be Empty Convert To String ${json}[blocks][1][offset]
+Check for datanodes
+ [arguments] ${datanodes} ${datanodes_expected}
+ Lists Should Be Equal ${datanodes} ${datanodes_expected} ignore_order=True
+
+Check for all datanodes
+ [arguments] ${json}
+ ${datanodes_expected} = Create List ozone_datanode_1.ozone_default ozone_datanode_2.ozone_default ozone_datanode_3.ozone_default
+ ${datanodes_b1} = Create List ${json}[blocks][0][replicas][0][hostname] ${json}[blocks][0][replicas][1][hostname] ${json}[blocks][0][replicas][2][hostname]
+ Check for datanodes ${datanodes_b1} ${datanodes_expected}
+ ${datanodes_b2} = Create List ${json}[blocks][1][replicas][0][hostname] ${json}[blocks][1][replicas][1][hostname] ${json}[blocks][1][replicas][2][hostname]
+ Check for datanodes ${datanodes_b2} ${datanodes_expected}
+
+Check checksum mismatch error
+ [arguments] ${json} ${datanode}
+ ${datanodes} = Create List ${json}[blocks][0][replicas][0][hostname] ${json}[blocks][0][replicas][1][hostname] ${json}[blocks][0][replicas][2][hostname]
+ ${index} = Get Index From List ${datanodes} ${datanode}
+ Should Contain ${json}[blocks][0][replicas][${index}][exception] Checksum mismatch
+
+Check unavailable datanode error
+ [arguments] ${json} ${datanode}
+ ${datanodes_b1} = Create List ${json}[blocks][0][replicas][0][hostname] ${json}[blocks][0][replicas][1][hostname] ${json}[blocks][0][replicas][2][hostname]
+ ${index_b1} = Get Index From List ${datanodes_b1} ${datanode}
+ Should Contain ${json}[blocks][0][replicas][${index_b1}][exception] UNAVAILABLE
+ ${datanodes_b2} = Create List ${json}[blocks][1][replicas][0][hostname] ${json}[blocks][1][replicas][1][hostname] ${json}[blocks][1][replicas][2][hostname]
+ ${index_b2} = Get Index From List ${datanodes_b2} ${datanode}
+ Should Contain ${json}[blocks][0][replicas][${index_b2}][exception] UNAVAILABLE
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org