You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/11/29 20:44:58 UTC
[42/50] [abbrv] hadoop git commit: HDDS-592. Fix ozone-secure.robot
test. Contributed by Ajay Kumar.
HDDS-592. Fix ozone-secure.robot test. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bbc95ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bbc95ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bbc95ee
Branch: refs/heads/HDDS-4
Commit: 8bbc95ee536caae21f28f9513fe624955aa122a0
Parents: 9128c66
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Nov 6 16:53:04 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:57:47 2018 -0800
----------------------------------------------------------------------
.../acceptance/ozone-secure.robot | 95 ----------------
.../dist/src/main/compose/ozonesecure/.env | 1 -
.../compose/ozonesecure/docker-compose.yaml | 22 ++--
.../ozonesecure/docker-image/runner/Dockerfile | 4 +-
.../main/smoketest/security/ozone-secure.robot | 111 +++++++++++++++++++
hadoop-ozone/dist/src/main/smoketest/test.sh | 2 +
6 files changed, 126 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
deleted file mode 100644
index 7fc1088..0000000
--- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
+++ /dev/null
@@ -1,95 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Smoke test to start cluster with docker-compose environments.
-Library OperatingSystem
-Suite Setup Startup Ozone Cluster
-Suite Teardown Teardown Ozone Cluster
-
-*** Variables ***
-${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root"
-${version}
-
-*** Test Cases ***
-
-Daemons are running
- Is daemon running om
- Is daemon running scm
- Is daemon running datanode
- Is daemon running ozone.kdc
-
-Check if datanode is connected to the scm
- Wait Until Keyword Succeeds 3min 5sec Have healthy datanodes 1
-
-Test rest interface
- ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
- Should contain ${result} 201 Created
- ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
- Should contain ${result} 201 Created
- ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
- Should contain ${result} 200 OK
- ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
- Should contain ${result} 200 OK
-
-Test ozone cli
- ${result} = Execute on 1 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
- Should contain ${result} Client cannot authenticate via
- # Authenticate testuser
- Execute on 0 datanode kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
- Execute on 0 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
- ${result} = Execute on 0 datanode ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
- Should contain ${result} createdOn
- Execute on 0 datanode ozone oz -updateVolume o3://om/hive -user bill -quota 10TB
- ${result} = Execute on 0 datanode ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
- Should Be Equal ${result} bill
-
-*** Keywords ***
-
-Startup Ozone Cluster
- ${rc} ${output} = Run docker compose 0 down
- ${rc} ${output} = Run docker compose 0 up -d
- Should Be Equal As Integers ${rc} 0
- Wait Until Keyword Succeeds 3min 10sec Is Daemon started ksm KSM is listening
-
-Teardown Ozone Cluster
- Run docker compose 0 down
-
-Is daemon running
- [arguments] ${name}
- ${result} = Run docker ps
- Should contain ${result} _${name}_1
-
-Is Daemon started
- [arguments] ${name} ${expression}
- ${rc} ${result} = Run docker compose 0 logs
- Should contain ${result} ${expression}
-
-Have healthy datanodes
- [arguments] ${requirednodes}
- ${result} = Execute on 0 scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value'
- Should Be Equal ${result} ${requirednodes}
-
-Execute on
- [arguments] ${expected_rc} ${componentname} ${command}
- ${rc} ${return} = Run docker compose ${expected_rc} exec ${componentname} ${command}
- [return] ${return}
-
-Run docker compose
- [arguments] ${expected_rc} ${command}
- Set Environment Variable OZONEDIR ${basedir}/hadoop-dist/target/ozone
- ${rc} ${output} = Run And Return Rc And Output docker-compose -f ${basedir}/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml ${command}
- Should Be Equal As Integers ${rc} ${expected_rc}
- [return] ${rc} ${output}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
index a494004..cac418a 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
@@ -15,4 +15,3 @@
# limitations under the License.
HDDS_VERSION=${hdds.version}
-SRC_VOLUME=../../
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
index fab5ba9..034d0ff 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -24,7 +24,7 @@ services:
buildno: 1
hostname: kdc
volumes:
- - $SRC_VOLUME:/opt/hadoop
+ - ../..:/opt/hadoop
datanode:
build:
context: docker-image/runner
@@ -32,13 +32,13 @@ services:
args:
buildno: 1
volumes:
- - $SRC_VOLUME:/opt/hadoop
+ - ../..:/opt/hadoop
hostname: datanode
ports:
- - 9864
+ - 9864
command: ["/opt/hadoop/bin/ozone","datanode"]
env_file:
- - docker-config
+ - docker-config
om:
build:
context: docker-image/runner
@@ -47,13 +47,13 @@ services:
buildno: 1
hostname: om
volumes:
- - $SRC_VOLUME:/opt/hadoop
+ - ../..:/opt/hadoop
ports:
- - 9874:9874
+ - 9874:9874
environment:
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
env_file:
- - docker-config
+ - docker-config
command: ["/opt/hadoop/bin/ozone","om"]
scm:
build:
@@ -63,11 +63,11 @@ services:
buildno: 1
hostname: scm
volumes:
- - $SRC_VOLUME:/opt/hadoop
+ - ../..:/opt/hadoop
ports:
- - 9876:9876
+ - 9876:9876
env_file:
- - docker-config
+ - docker-config
environment:
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
+ command: ["/opt/hadoop/bin/ozone","scm"]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
index efda03e..17eff18 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
@@ -15,8 +15,8 @@
# limitations under the License.
FROM openjdk:8-jdk
-RUN apt-get update && apt-get install -y jq curl python sudo && apt-get clean
-
+RUN apt-get update && apt-get install -y jq curl python python-pip sudo && apt-get clean
+RUN pip install robotframework
RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
RUN chmod +x /usr/local/bin/dumb-init
RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot
new file mode 100644
index 0000000..457ebaa
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Smoke test to start cluster with docker-compose environments.
+Library OperatingSystem
+Resource ../commonlib.robot
+
+*** Test Cases ***
+Create volume and bucket
+ ${rc} ${output} = Run And Return Rc And Output ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root
+ Should contain ${output} Client cannot authenticate via
+ # Authenticate testuser
+ Execute kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
+ Execute ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root
+ Execute ozone sh volume create o3://om/fstest2 --user bilbo --quota 100TB --root
+ Execute ozone sh bucket create o3://om/fstest/bucket1
+ Execute ozone sh bucket create o3://om/fstest/bucket2
+ Execute ozone sh bucket create o3://om/fstest2/bucket3
+
+Check volume from ozonefs
+ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/
+
+Run ozoneFS tests
+ Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} testdir/deep
+ Execute ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} NOTICE.txt
+
+ Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} PUTFILE.txt
+
+ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/
+ Should contain ${result} NOTICE.txt
+ Should contain ${result} PUTFILE.txt
+
+ Execute ozone fs -mv o3fs://bucket1.fstest/testdir/deep/NOTICE.txt o3fs://bucket1.fstest/testdir/deep/MOVED.TXT
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} MOVED.TXT
+ Should not contain ${result} NOTICE.txt
+
+ Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep/subdir1
+ Execute ozone fs -cp o3fs://bucket1.fstest/testdir/deep/MOVED.TXT o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} subdir1/NOTICE.txt
+
+ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/subdir1/
+ Should contain ${result} NOTICE.txt
+
+ Execute ozone fs -cat o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+ Should not contain ${result} Failed
+
+ Execute ozone fs -rm o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should not contain ${result} NOTICE.txt
+
+ ${result} = Execute ozone fs -rmdir o3fs://bucket1.fstest/testdir/deep/subdir1/
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should not contain ${result} subdir1
+
+ Execute ozone fs -touch o3fs://bucket1.fstest/testdir/TOUCHFILE.txt
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} TOUCHFILE.txt
+
+ Execute ozone fs -rm -r o3fs://bucket1.fstest/testdir/
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should not contain ${result} testdir
+
+ Execute rm -Rf localdir1
+ Execute mkdir localdir1
+ Execute cp NOTICE.txt localdir1/LOCAL.txt
+ Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir1
+ Execute ozone fs -copyFromLocal localdir1 o3fs://bucket1.fstest/testdir1/
+ Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir1/NOTICE.txt
+
+ ${result} = Execute ozone fs -ls -R o3fs://bucket1.fstest/testdir1/
+ Should contain ${result} localdir1/LOCAL.txt
+ Should contain ${result} testdir1/NOTICE.txt
+
+ Execute ozone fs -mkdir -p o3fs://bucket2.fstest/testdir2
+ Execute ozone fs -mkdir -p o3fs://bucket3.fstest2/testdir3
+ Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket2.fstest/testdir2/
+ Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket3.fstest2/testdir3/
+ Execute ozone sh key put o3://om/fstest/bucket1/KEY.txt NOTICE.txt
+ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/KEY.txt
+ Should contain ${result} KEY.txt
+ ${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/KEY.txt
+ Should Be Equal As Integers ${rc} 1
+ Should contain ${result} File exists
+ Execute rm -Rf GET.txt
+ Execute ozone fs -get o3fs://bucket1.fstest/KEY.txt GET.txt
+ Execute ls -l GET.txt
+ ${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3fs://abcde.pqrs/
+ Should Be Equal As Integers ${rc} 1
+ Should contain ${result} VOLUME_NOT_FOUND
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/smoketest/test.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/test.sh b/hadoop-ozone/dist/src/main/smoketest/test.sh
index 5819595..53d0876 100755
--- a/hadoop-ozone/dist/src/main/smoketest/test.sh
+++ b/hadoop-ozone/dist/src/main/smoketest/test.sh
@@ -107,6 +107,8 @@ if [ "$RUN_ALL" = true ]; then
#
# We select the test suites and execute them on multiple type of clusters
#
+ DEFAULT_TESTS=("security")
+ execute_tests ozonesecure "${DEFAULT_TESTS[@]}"
DEFAULT_TESTS=("basic")
execute_tests ozone "${DEFAULT_TESTS[@]}"
TESTS=("ozonefs")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org