[ozone] branch master updated: HDDS-4760. Intermittent failure in ozone-ha acceptance test (#1886)

[ad...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git


The following commit(s) were added to refs/heads/master by this push:
     new e712a1f  HDDS-4760. Intermittent failure in ozone-ha acceptance test (#1886)
e712a1f is described below

commit e712a1fc761850122843ad73a7479a783d42f2b0
Author: Doroszlai, Attila <64...@users.noreply.github.com>
AuthorDate: Sat Feb 6 08:47:16 2021 +0100

    HDDS-4760. Intermittent failure in ozone-ha acceptance test (#1886)
---
 .../dist/src/main/compose/ozone-ha/test.sh         |  1 +
 hadoop-ozone/dist/src/main/compose/testlib.sh      | 40 +++++++++++++++++++---
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
index 51bff59..5913b65 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
@@ -20,6 +20,7 @@ export COMPOSE_DIR
 
 export SECURITY_ENABLED=false
 export OZONE_REPLICATION_FACTOR=3
+export OM_SERVICE_ID=omservice
 
 # shellcheck source=/dev/null
 source "$COMPOSE_DIR/../testlib.sh"
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 105ec56..0e12fde 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -23,8 +23,6 @@ RESULT_DIR_INSIDE="/tmp/smoketest/$(basename "$COMPOSE_ENV_NAME")/result"
 OM_HA_PARAM=""
 if [[ -n "${OM_SERVICE_ID}" ]] && [[ "${OM_SERVICE_ID}" != "om" ]]; then
   OM_HA_PARAM="--om-service-id=${OM_SERVICE_ID}"
-else
-  OM_SERVICE_ID=om
 fi
 
 ## @description create results directory, purging any prior data
@@ -95,6 +93,39 @@ wait_for_safemode_exit(){
    return 1
 }
 
+## @description wait until OM leader is elected (or 120 seconds)
+wait_for_om_leader() {
+  if [[ -z "${OM_SERVICE_ID:-}" ]]; then
+    echo "No OM HA service, no need to wait"
+    return
+  fi
+
+  #Reset the timer
+  SECONDS=0
+
+  #Don't give it up until 120 seconds
+  while [[ $SECONDS -lt 120 ]]; do
+    local command="ozone admin om roles --service-id '${OM_SERVICE_ID}'"
+    if [[ "${SECURITY_ENABLED}" == 'true' ]]; then
+      status=$(docker-compose exec -T scm bash -c "kinit -k scm/scm@EXAMPLE.COM -t /etc/security/keytabs/scm.keytab && $command" | grep LEADER)
+    else
+      status=$(docker-compose exec -T scm bash -c "$command" | grep LEADER)
+    fi
+    if [[ -n "${status}" ]]; then
+      echo "Found OM leader for service ${OM_SERVICE_ID}: $status"
+      return
+    else
+      echo "Waiting for OM leader for service ${OM_SERVICE_ID}"
+    fi
+
+    echo "SECONDS: $SECONDS"
+
+    sleep 2
+  done
+  echo "WARNING: OM leader still not found for service ${OM_SERVICE_ID}"
+  return 1
+}
+
 ## @description  Starts a docker-compose based test environment
 ## @param number of datanodes to start and wait for (default: 3)
 start_docker_env(){
@@ -104,7 +135,8 @@ start_docker_env(){
   export OZONE_SAFEMODE_MIN_DATANODES="${datanode_count}"
   docker-compose --no-ansi down
   if ! { docker-compose --no-ansi up -d --scale datanode="${datanode_count}" \
-      && wait_for_safemode_exit ; }; then
+      && wait_for_safemode_exit \
+      && wait_for_om_leader ; }; then
     OUTPUT_NAME="$COMPOSE_ENV_NAME"
     stop_docker_env
     return 1
@@ -142,7 +174,7 @@ execute_robot_test(){
     && docker-compose exec -T "$CONTAINER" robot \
       -v KEY_NAME:"${OZONE_BUCKET_KEY_NAME}" \
       -v OM_HA_PARAM:"${OM_HA_PARAM}" \
-      -v OM_SERVICE_ID:"${OM_SERVICE_ID}" \
+      -v OM_SERVICE_ID:"${OM_SERVICE_ID:-om}" \
       -v OZONE_DIR:"${OZONE_DIR}" \
       -v SECURITY_ENABLED:"${SECURITY_ENABLED}" \
       ${ARGUMENTS[@]} --log NONE --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$OUTPUT_PATH" \


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org