You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2019/10/13 06:50:30 UTC
[hadoop-ozone] 01/01: HDDS-2278. Run S3 test suite on OM HA cluste.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-2278
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit c75e74cab06a61de7c2ae7961337a34e19dd3412
Author: Bharat Viswanadham <bh...@apache.org>
AuthorDate: Thu Oct 10 17:35:30 2019 -0700
HDDS-2278. Run S3 test suite on OM HA cluste.
---
.../{ozones3/test.sh => ozone-om-ha-s3/.env} | 18 +----
.../compose/ozone-om-ha-s3/docker-compose.yaml | 89 ++++++++++++++++++++++
.../src/main/compose/ozone-om-ha-s3/docker-config | 34 +++++++++
.../compose/{ozones3 => ozone-om-ha-s3}/test.sh | 4 +-
hadoop-ozone/dist/src/main/compose/ozones3/test.sh | 2 +
.../dist/src/main/smoketest/s3/commonawslib.robot | 13 ++--
.../main/smoketest/security/ozone-secure-s3.robot | 2 +-
7 files changed, 137 insertions(+), 25 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/.env
old mode 100755
new mode 100644
similarity index 71%
copy from hadoop-ozone/dist/src/main/compose/ozones3/test.sh
copy to hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/.env
index 0160da9..96ab163
--- a/hadoop-ozone/dist/src/main/compose/ozones3/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/.env
@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,18 +14,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export COMPOSE_DIR
-
-# shellcheck source=/dev/null
-source "$COMPOSE_DIR/../testlib.sh"
-
-start_docker_env
-
-execute_robot_test scm basic/basic.robot
-
-execute_robot_test scm s3
-
-stop_docker_env
-
-generate_report
+HDDS_VERSION=${hdds.version}
+OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-compose.yaml
new file mode 100644
index 0000000..390acb3
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-compose.yaml
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ datanode:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ privileged: true #required by the profiler
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ om1:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ privileged: true #required by the profiler
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9880:9874
+ - 9890:9872
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ om2:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ privileged: true #required by the profiler
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9882:9874
+ - 9892:9872
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ om3:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ privileged: true #required by the profiler
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9884:9874
+ - 9894:9872
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ scm:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ privileged: true #required by the profiler
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
+ s3g:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9878:9878
+ env_file:
+ - ./docker-config
+ command: ["ozone","s3g"]
+
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config
new file mode 100644
index 0000000..646e875
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
+CORE-SITE.XML_fs.defaultFS=o3fs://bucket.volume.id1
+OZONE-SITE.XML_ozone.om.service.ids=id1
+OZONE-SITE.XML_ozone.om.nodes.id1=om1,om2,om3
+OZONE-SITE.XML_ozone.om.address.id1.om1=om1
+OZONE-SITE.XML_ozone.om.address.id1.om2=om2
+OZONE-SITE.XML_ozone.om.address.id1.om3=om3
+OZONE-SITE.XML_ozone.om.ratis.enable=true
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.replication=1
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/test.sh
old mode 100755
new mode 100644
similarity index 96%
copy from hadoop-ozone/dist/src/main/compose/ozones3/test.sh
copy to hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/test.sh
index 0160da9..238e757
--- a/hadoop-ozone/dist/src/main/compose/ozones3/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/test.sh
@@ -18,13 +18,13 @@
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export COMPOSE_DIR
+export SECURITY_ENABLED=false
+
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
start_docker_env
-execute_robot_test scm basic/basic.robot
-
execute_robot_test scm s3
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/test.sh b/hadoop-ozone/dist/src/main/compose/ozones3/test.sh
index 0160da9..a534a62 100755
--- a/hadoop-ozone/dist/src/main/compose/ozones3/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozones3/test.sh
@@ -18,6 +18,8 @@
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export COMPOSE_DIR
+export SECURITY_ENABLED=false
+
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../testlib.sh"
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
index 1335635..90b778a 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
@@ -50,21 +50,22 @@ Setup v2 headers
Setup v4 headers
Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab
+ Run Keyword if '${SECURITY_ENABLED}' == 'true' Setup secure v4 headers
+ Run Keyword if '${SECURITY_ENABLED}' == 'false' Setup dummy credentials for S3
+
+Setup secure v4 headers
${result} = Execute ozone s3 getsecret
${accessKey} = Get Regexp Matches ${result} (?<=awsAccessKey=).*
${accessKey} = Get Variable Value ${accessKey} sdsdasaasdasd
${secret} = Get Regexp Matches ${result} (?<=awsSecret=).*
-
- ${len}= Get Length ${accessKey}
- ${accessKey}= Set Variable If ${len} > 0 ${accessKey[0]} kljdfslff
- ${len}= Get Length ${secret}
- ${secret}= Set Variable If ${len} > 0 ${secret[0]} dhafldhlf
+ ${accessKey} = Set Variable ${accessKey[0]}
+ ${secret} = Set Variable ${secret[0]}
Execute aws configure set default.s3.signature_version s3v4
Execute aws configure set aws_access_key_id ${accessKey}
Execute aws configure set aws_secret_access_key ${secret}
Execute aws configure set region us-west-1
-Setup incorrect credentials for S3
+Setup dummy credentials for S3
Execute aws configure set default.s3.signature_version s3v4
Execute aws configure set aws_access_key_id dlfknslnfslf
Execute aws configure set aws_secret_access_key dlfknslnfslf
diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-s3.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-s3.robot
index 90166fe..9f5e122 100644
--- a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-s3.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-s3.robot
@@ -38,7 +38,7 @@ Secure S3 test Success
Should contain ${output} bucket-test123
Secure S3 test Failure
- Run Keyword Setup incorrect credentials for S3
+ Run Keyword Setup dummy credentials for S3
${rc} ${result} = Run And Return Rc And Output aws s3api --endpoint-url ${ENDPOINT_URL} create-bucket --bucket bucket-test123
Should Be True ${rc} > 0
---------------------------------------------------------------------
To unsubscribe, e-mail: hdfs-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: hdfs-commits-help@hadoop.apache.org