You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ms...@apache.org on 2021/07/19 01:22:10 UTC
[ozone] branch master updated: HDDS-5436. Add acceptance test for
Hadoop 3.3 (#2428)
This is an automated email from the ASF dual-hosted git repository.
msingh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b83039e HDDS-5436. Add acceptance test for Hadoop 3.3 (#2428)
b83039e is described below
commit b83039eefe36d42f762b08662475c09d1c86819b
Author: Doroszlai, Attila <64...@users.noreply.github.com>
AuthorDate: Mon Jul 19 03:21:53 2021 +0200
HDDS-5436. Add acceptance test for Hadoop 3.3 (#2428)
---
.../{ozonesecure-mr => ozone-mr/hadoop33}/.env | 7 +-
.../compose/ozone-mr/hadoop33/docker-compose.yaml | 110 +++++++++++++++++++++
.../.env => ozone-mr/hadoop33/docker-config} | 14 +--
.../.env => ozone-mr/hadoop33/test.sh} | 31 ++++--
.../dist/src/main/compose/ozonesecure-mr/.env | 2 +-
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 2 +-
6 files changed, 148 insertions(+), 18 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/.env
similarity index 84%
copy from hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
copy to hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/.env
index 2490b1f..74130cb 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/.env
@@ -14,9 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-HDDS_VERSION=${hdds.version}
+HDDS_VERSION=@hdds.version@
HADOOP_IMAGE=flokkr/hadoop
-HADOOP_VERSION=3.2.2
-OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
-OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
+HADOOP_VERSION=3.3.1
+OZONE_RUNNER_VERSION=@docker.ozone-runner.version@
OZONE_OPTS=
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/docker-compose.yaml
new file mode 100644
index 0000000..c07282a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/docker-compose.yaml
@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ datanode:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../../..:/opt/hadoop
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - docker-config
+ - ../common-config
+ om:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: om
+ volumes:
+ - ../../..:/opt/hadoop
+ ports:
+ - 9874:9874
+ - 9862:9862
+ environment:
+ WAITFOR: scm:9876
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ OZONE_OPTS:
+ env_file:
+ - docker-config
+ - ../common-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ s3g:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: s3g
+ volumes:
+ - ../../..:/opt/hadoop
+ ports:
+ - 9878:9878
+ env_file:
+ - ./docker-config
+ - ../common-config
+ environment:
+ OZONE_OPTS:
+ command: ["/opt/hadoop/bin/ozone","s3g"]
+ scm:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: scm
+ volumes:
+ - ../../..:/opt/hadoop
+ ports:
+ - 9876:9876
+ - 9860:9860
+ env_file:
+ - docker-config
+ - ../common-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}"
+ OZONE_OPTS:
+ command: ["/opt/hadoop/bin/ozone","scm"]
+ rm:
+ image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+ hostname: rm
+ volumes:
+ - ../../..:/opt/ozone
+ - ../../../libexec/transformation.py:/opt/transformation.py
+ ports:
+ - 8088:8088
+ env_file:
+ - ./docker-config
+ - ../common-config
+ command: ["yarn", "resourcemanager"]
+ nm:
+ image: ${HADOOP_IMAGE}:${HADOOP_VERSION}
+ hostname: nm
+ volumes:
+ - ../../..:/opt/ozone
+ - ../../../libexec/transformation.py:/opt/transformation.py
+ env_file:
+ - ./docker-config
+ - ../common-config
+ environment:
+ WAITFOR: rm:8088
+ command: ["yarn","nodemanager"]
+# Optional section: comment out this part to get DNS resolution for all the containers.
+# Add 127.0.0.1 (or the ip of your docker machine) to the resolv.conf to get local DNS resolution
+# For all the containers (including resource managers and Node manager UI)
+# dns:
+# image: andyshinn/dnsmasq:2.76
+# ports:
+# - 53:53/udp
+# - 53:53/tcp
+# volumes:
+# - "/var/run/docker.sock:/var/run/docker.sock"
+# command:
+# - "-k"
+# - "-d"
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/docker-config
similarity index 60%
copy from hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
copy to hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/docker-config
index 2490b1f..d2fa492 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/docker-config
@@ -14,9 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-HDDS_VERSION=${hdds.version}
-HADOOP_IMAGE=flokkr/hadoop
-HADOOP_VERSION=3.2.2
-OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
-OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
-OZONE_OPTS=
+CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
+CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs
+MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/ozone-filesystem-hadoop3-@project.version@.jar
+
+HADOOP_CLASSPATH=/opt/ozone/share/ozone/lib/ozone-filesystem-hadoop3-@project.version@.jar
+OZONE_CLASSPATH=
+
+no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/test.sh
old mode 100644
new mode 100755
similarity index 55%
copy from hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
copy to hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/test.sh
index 2490b1f..9e0cea4
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop33/test.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,9 +15,27 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-HDDS_VERSION=${hdds.version}
-HADOOP_IMAGE=flokkr/hadoop
-HADOOP_VERSION=3.2.2
-OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
-OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
-OZONE_OPTS=
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../../testlib.sh"
+
+start_docker_env
+
+execute_robot_test scm createmrenv.robot
+
+# reinitialize the directories to use
+export OZONE_DIR=/opt/ozone
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../../testlib.sh"
+
+for scheme in o3fs ofs; do
+ execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
+ execute_robot_test rm -v "SCHEME:${scheme}" -N "mapreduce-${scheme}" mapreduce.robot
+done
+
+stop_docker_env
+
+generate_report
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
index 2490b1f..4b4a913 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/.env
@@ -16,7 +16,7 @@
HDDS_VERSION=${hdds.version}
HADOOP_IMAGE=flokkr/hadoop
-HADOOP_VERSION=3.2.2
+HADOOP_VERSION=3.3.1
OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
OZONE_OPTS=
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
index bee946e..a2779d9 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
@@ -36,4 +36,4 @@ Test hadoop dfs
${result} = Execute hdfs dfs -ls ${dir}
Should contain ${result} ${PREFIX}-${random}
${result} = Execute hdfs dfs -cat ${dir}/${PREFIX}-${random}
- Should contain ${result} This product includes software developed by The Apache Software
\ No newline at end of file
+ Should contain ${result} This product includes software developed
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org