You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2020/09/03 14:43:48 UTC
[hadoop-ozone] branch master updated: HDDS-3840. Use OFS in
MapReduce acceptance test (#1365)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ead6371 HDDS-3840. Use OFS in MapReduce acceptance test (#1365)
ead6371 is described below
commit ead6371b584b7246ed1304782c16f561a46b0356
Author: Doroszlai, Attila <64...@users.noreply.github.com>
AuthorDate: Thu Sep 3 16:43:31 2020 +0200
HDDS-3840. Use OFS in MapReduce acceptance test (#1365)
---
.../dist/src/main/compose/ozone-mr/hadoop27/test.sh | 7 ++++---
.../dist/src/main/compose/ozone-mr/hadoop31/test.sh | 7 ++++---
.../dist/src/main/compose/ozone-mr/hadoop32/test.sh | 7 ++++---
hadoop-ozone/dist/src/main/compose/ozonesecure-mr/test.sh | 5 ++++-
hadoop-ozone/dist/src/main/smoketest/createmrenv.robot | 13 ++++++++++---
hadoop-ozone/dist/src/main/smoketest/mapreduce.robot | 11 +++++++++--
.../dist/src/main/smoketest/ozonefs/hadoopo3fs.robot | 12 ++++++++----
7 files changed, 43 insertions(+), 19 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/test.sh
index 832bb46..20d87cd 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/test.sh
@@ -30,9 +30,10 @@ export OZONE_DIR=/opt/ozone
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../../testlib.sh"
-execute_robot_test rm ozonefs/hadoopo3fs.robot
-
-execute_robot_test rm -v hadoop.version:2.7.3 mapreduce.robot
+for scheme in o3fs ofs; do
+ execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
+ execute_robot_test rm -v "hadoop.version:${HADOOP_VERSION}" -v "SCHEME:${scheme}" -N "mapreduce-${scheme}" mapreduce.robot
+done
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh
index 362ff98..08d19b8 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/test.sh
@@ -34,9 +34,10 @@ export OZONE_DIR=/opt/ozone
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../../testlib.sh"
-execute_robot_test rm ozonefs/hadoopo3fs.robot
-
-execute_robot_test rm -v "hadoop.version:${HADOOP_VERSION}" mapreduce.robot
+for scheme in o3fs ofs; do
+ execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
+ execute_robot_test rm -v "hadoop.version:${HADOOP_VERSION}" -v "SCHEME:${scheme}" -N "mapreduce-${scheme}" mapreduce.robot
+done
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/test.sh
index b1910a5..9e0cea4 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/test.sh
@@ -31,9 +31,10 @@ export OZONE_DIR=/opt/ozone
# shellcheck source=/dev/null
source "$COMPOSE_DIR/../../testlib.sh"
-execute_robot_test rm ozonefs/hadoopo3fs.robot
-
-execute_robot_test rm mapreduce.robot
+for scheme in o3fs ofs; do
+ execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
+ execute_robot_test rm -v "SCHEME:${scheme}" -N "mapreduce-${scheme}" mapreduce.robot
+done
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/test.sh
index cc6ebf0..3763397 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/test.sh
@@ -37,7 +37,10 @@ source "$COMPOSE_DIR/../testlib.sh"
execute_robot_test rm kinit-hadoop.robot
-execute_robot_test rm mapreduce.robot
+for scheme in o3fs ofs; do
+ execute_robot_test rm -v "SCHEME:${scheme}" -N "hadoopfs-${scheme}" ozonefs/hadoopo3fs.robot
+ execute_robot_test rm -v "SCHEME:${scheme}" -N "mapreduce-${scheme}" mapreduce.robot
+done
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot b/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot
index 579fdde..d36db58 100644
--- a/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/createmrenv.robot
@@ -17,10 +17,12 @@
Documentation Create directories required for MR test
Library OperatingSystem
Resource commonlib.robot
+Resource lib/fs.robot
Test Timeout 2 minute
*** Variables ***
+${SCHEME} o3fs
${volume} volume1
${bucket} bucket1
@@ -40,8 +42,13 @@ Create test volume, bucket and key
Run Keyword if "BUCKET_NOT_FOUND" in """${result}""" Create bucket
${result} = Execute ozone sh bucket info /${volume}/${bucket}
Should not contain ${result} NOT_FOUND
- Execute ozone sh key put /volume1/bucket1/key1 LICENSE.txt
+ ${dir} = Format FS URL ${SCHEME} ${volume} ${bucket} input
+ Execute ozone fs -mkdir ${dir}
+ Execute ozone sh key put /${volume}/${bucket}/input/key1 LICENSE.txt
+ Execute ozone sh key put /${volume}/${bucket}/input/key2 NOTICE.txt
+ Execute ozone sh key put /${volume}/${bucket}/input/key3 README.md
Create user dir for hadoop
- Execute ozone fs -mkdir /user
- Execute ozone fs -mkdir /user/hadoop
+ ${dir} = Format FS URL ${SCHEME} ${volume} ${bucket} user
+ Execute ozone fs -mkdir ${dir}
+ Execute ozone fs -mkdir ${dir}/hadoop
diff --git a/hadoop-ozone/dist/src/main/smoketest/mapreduce.robot b/hadoop-ozone/dist/src/main/smoketest/mapreduce.robot
index 654dd49..9eff89e 100644
--- a/hadoop-ozone/dist/src/main/smoketest/mapreduce.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/mapreduce.robot
@@ -17,10 +17,12 @@
Documentation Execute MR jobs
Library OperatingSystem
Resource commonlib.robot
+Resource lib/fs.robot
Test Timeout 4 minute
*** Variables ***
+${SCHEME} o3fs
${volume} volume1
${bucket} bucket1
@@ -33,11 +35,16 @@ Find example jar
Execute PI calculation
${exampleJar} Find example jar
- ${output} = Execute yarn jar ${exampleJar} pi 3 3
+ ${root} = Format FS URL ${SCHEME} ${volume} ${bucket}
+ ${output} = Execute yarn jar ${exampleJar} pi -D fs.defaultFS=${root} 3 3
Should Contain ${output} completed successfully
Execute WordCount
${exampleJar} Find example jar
${random} Generate Random String 2 [NUMBERS]
- ${output} = Execute yarn jar ${exampleJar} wordcount o3fs://bucket1.volume1/key1 o3fs://bucket1.volume1/key1-${random}.count
+ ${root} = Format FS URL ${SCHEME} ${volume} ${bucket}
+ ${dir} = Format FS URL ${SCHEME} ${volume} ${bucket} input/
+ ${result} = Format FS URL ${SCHEME} ${volume} ${bucket} wordcount-${random}.txt
+ ${output} = Execute yarn jar ${exampleJar} wordcount -D fs.defaultFS=${root} ${dir} ${result}
+ Should Contain ${output} map tasks=3
Should Contain ${output} completed successfully
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
index df9f3af..adcb943 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
@@ -18,16 +18,20 @@ Documentation Test ozone fs with hadoopfs
Library OperatingSystem
Library String
Resource ../commonlib.robot
+Resource ../lib/fs.robot
Test Timeout 5 minutes
*** Variables ***
-${DATANODE_HOST} datanode
-${PREFIX} ozone
+${SCHEME} o3fs
+${volume} volume1
+${bucket} bucket1
+${PREFIX} ozone
*** Test cases ***
Test hadoop dfs
+ ${dir} = Format FS URL ${SCHEME} ${volume} ${bucket}
${random} = Generate Random String 5 [NUMBERS]
- ${result} = Execute hdfs dfs -put /opt/hadoop/NOTICE.txt o3fs://bucket1.volume1/${PREFIX}-${random}
- ${result} = Execute hdfs dfs -ls o3fs://bucket1.volume1/
+ ${result} = Execute hdfs dfs -put /opt/hadoop/NOTICE.txt ${dir}/${PREFIX}-${random}
+ ${result} = Execute hdfs dfs -ls ${dir}
Should contain ${result} ${PREFIX}-${random}
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org