You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by pi...@apache.org on 2021/11/29 16:49:46 UTC
[ozone] branch HDDS-5447-httpfs updated: HDDS-5698 [HTTPFSGW] Port HTTPFS node and robot tests to ozone-ha, and ozonesecure(-ha) (#2861)
This is an automated email from the ASF dual-hosted git repository.
pifta pushed a commit to branch HDDS-5447-httpfs
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-5447-httpfs by this push:
new 675e0b9 HDDS-5698 [HTTPFSGW] Port HTTPFS node and robot tests to ozone-ha, and ozonesecure(-ha) (#2861)
675e0b9 is described below
commit 675e0b90856a7fb666838147c07941b3e4d62bf2
Author: Zita Dombi <50...@users.noreply.github.com>
AuthorDate: Mon Nov 29 17:49:28 2021 +0100
HDDS-5698 [HTTPFSGW] Port HTTPFS node and robot tests to ozone-ha, and ozonesecure(-ha) (#2861)
---
hadoop-ozone/dist/pom.xml | 2 +-
.../src/main/compose/ozone-ha/docker-compose.yaml | 9 ++++++
.../dist/src/main/compose/ozone-ha/docker-config | 4 +++
.../dist/src/main/compose/ozone-ha/test.sh | 1 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +-
.../compose/ozonesecure-ha/docker-compose.yaml | 19 +++++++++++++
.../src/main/compose/ozonesecure-ha/docker-config | 21 ++++++++++++--
.../dist/src/main/compose/ozonesecure-ha/test.sh | 2 ++
.../main/compose/ozonesecure/docker-compose.yaml | 17 ++++++++++-
.../src/main/compose/ozonesecure/docker-config | 18 +++++++++++-
.../dist/src/main/compose/ozonesecure/test.sh | 2 ++
hadoop-ozone/dist/src/main/keytabs/HTTP.keytab | Bin 432 -> 580 bytes
hadoop-ozone/dist/src/main/keytabs/dn.keytab | Bin 278 -> 278 bytes
hadoop-ozone/dist/src/main/keytabs/hadoop.keytab | Bin 146 -> 146 bytes
hadoop-ozone/dist/src/main/keytabs/httpfs.keytab | Bin 0 -> 458 bytes
hadoop-ozone/dist/src/main/keytabs/jhs.keytab | Bin 142 -> 142 bytes
hadoop-ozone/dist/src/main/keytabs/nm.keytab | Bin 138 -> 138 bytes
hadoop-ozone/dist/src/main/keytabs/om.keytab | Bin 278 -> 278 bytes
hadoop-ozone/dist/src/main/keytabs/recon.keytab | Bin 296 -> 296 bytes
hadoop-ozone/dist/src/main/keytabs/rm.keytab | Bin 138 -> 138 bytes
hadoop-ozone/dist/src/main/keytabs/s3g.keytab | Bin 434 -> 434 bytes
hadoop-ozone/dist/src/main/keytabs/scm.keytab | Bin 586 -> 586 bytes
hadoop-ozone/dist/src/main/keytabs/testuser.keytab | Bin 450 -> 606 bytes
.../dist/src/main/keytabs/testuser2.keytab | Bin 154 -> 154 bytes
.../dist/src/main/keytabs/update-keytabs.sh | 6 ++++
.../src/main/smoketest/httpfs/operations.robot | 19 +++++++------
.../main/smoketest/httpfs/operations_tests.robot | 31 +++++++++++++++++----
27 files changed, 133 insertions(+), 20 deletions(-)
diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml
index d487544..2783d06 100644
--- a/hadoop-ozone/dist/pom.xml
+++ b/hadoop-ozone/dist/pom.xml
@@ -29,7 +29,7 @@
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
<docker.ozone-runner.version>20210329-1</docker.ozone-runner.version>
- <docker.ozone-testkr5b.image>apache/ozone-testkrb5:20210419-1</docker.ozone-testkr5b.image>
+ <docker.ozone-testkr5b.image>apache/ozone-testkrb5:20211102-1</docker.ozone-testkr5b.image>
</properties>
<build>
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-compose.yaml
index 615dc9e..a337243 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-compose.yaml
@@ -100,6 +100,15 @@ services:
OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1}
<<: *replication
command: ["ozone","scm"]
+ httpfs:
+ <<: *common-config
+ environment:
+ OZONE_CLASSPATH: "/opt/hadoop/share/ozone/lib/ozone-filesystem-1.2.0-SNAPSHOT.jar:/opt/hadoop/share/ozone/lib/ozone-filesystem-common-1.2.0-SNAPSHOT.jar:/opt/hadoop/share/ozone/lib/ozone-filesystem-hadoop3-1.2.0-SNAPSHOT.jar"
+ OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1}
+ <<: *replication
+ ports:
+ - 14000:14000
+ command: [ "ozone","httpfs" ]
s3g:
<<: *common-config
environment:
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config
index 5b2632d..c125dbe 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config
@@ -14,6 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# For HttpFS service it is required to enable proxying users.
+CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=*
+CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=*
+
CORE-SITE.XML_fs.defaultFS=ofs://omservice/
OZONE-SITE.XML_ozone.om.service.ids=omservice
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
index 3a21ef4..2f10ba7 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
@@ -34,6 +34,7 @@ execute_robot_test ${SCM} basic/ozone-shell-single.robot
execute_robot_test ${SCM} basic/links.robot
execute_robot_test ${SCM} s3
execute_robot_test ${SCM} freon
+execute_robot_test ${SCM} -v SECURITY_ENABLED:${SECURITY_ENABLED} -v USERNAME:httpfs httpfs
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
index b1f1d07..91b88ec 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
@@ -50,7 +50,7 @@ execute_robot_test scm freon
execute_robot_test scm cli
execute_robot_test scm admincli
-execute_robot_test scm httpfs
+execute_robot_test scm -v SECURITY_ENABLED:${SECURITY_ENABLED} -v USERNAME:httpfs httpfs
execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-simple-ofs-bucket ozonefs/ozonefs.robot
execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-simple-o3fs-link ozonefs/ozonefs.robot
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml
index 6a02661..684fef7 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml
@@ -183,6 +183,25 @@ services:
networks:
ozone_net:
ipv4_address: 172.25.0.113
+ httpfs:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: httpfs
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ ports:
+ - 14000:14000
+ env_file:
+ - ./docker-config
+ command: [ "/opt/hadoop/bin/ozone","httpfs" ]
+ environment:
+ OZONE_CLASSPATH: "/opt/hadoop/share/ozone/lib/ozone-filesystem-1.2.0-SNAPSHOT.jar:/opt/hadoop/share/ozone/lib/ozone-filesystem-common-1.2.0-SNAPSHOT.jar:/opt/hadoop/share/ozone/lib/ozone-filesystem-hadoop3-1.2.0-SNAPSHOT.jar"
+ OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1}
+ OZONE_OPTS:
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.119
s3g:
image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
hostname: s3g
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
index 91e80d3..f09b2b7 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
@@ -14,6 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# For HttpFS it is required to enable impersonating users as root, because the httpfs is mapped to root as a result of the auth_to_local rules.
+CORE-SITE.XML_hadoop.proxyuser.root.hosts=*
+CORE-SITE.XML_hadoop.proxyuser.root.groups=*
+
CORE-SITE.XML_fs.defaultFS=ofs://id1
OZONE-SITE.XML_ozone.om.service.ids=id1
@@ -60,14 +64,14 @@ OZONE-SITE.XML_ozone.recon.address=recon:9891
OZONE-SITE.XML_ozone.security.enabled=true
OZONE-SITE.XML_ozone.acl.enabled=true
OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer
-OZONE-SITE.XML_ozone.administrators="testuser/scm@EXAMPLE.COM,testuser/s3g@EXAMPLE.COM,recon/recon@EXAMPLE.COM,om/om1@EXAMPLE.COM,om/om2@EXAMPLE.COM,om/om3@EXAMPLE.COM"
+OZONE-SITE.XML_ozone.administrators="testuser/scm@EXAMPLE.COM,testuser/s3g@EXAMPLE.COM,testuser/httpfs@EXAMPLE.COM,recon/recon@EXAMPLE.COM,om/om1@EXAMPLE.COM,om/om2@EXAMPLE.COM,om/om3@EXAMPLE.COM"
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
CORE-SITE.XML_dfs.data.transfer.protection=authentication
CORE-SITE.XML_hadoop.security.authentication=kerberos
-CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1](testuser2.*) RULE:[2:$1@$0](.*)s/.*/root/
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1](testuser2.*) RULE:[2:$1](testuser.*) RULE:[2:$1@$0](.*)s/.*/root/
CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms
@@ -81,6 +85,9 @@ OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM
OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM
+OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab
+OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM
+
HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM
HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
@@ -94,6 +101,7 @@ OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos
OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos
OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
@@ -104,6 +112,8 @@ OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/db@EXAMPLE.COM
OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
+OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
+OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM
OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=HTTP/recon@EXAMPLE.COM
OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab
OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab
@@ -125,6 +135,13 @@ HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+HTTPFS-SITE.XML_hadoop.http.authentication.type=kerberos
+HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
+HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/httpfs@EXAMPLE.COM
+HTTPFS-SITE.XML_httpfs.hadoop.authentication.type=kerberos
+HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
+HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal=httpfs/httpfs@EXAMPLE.COM
+
#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
index 252f953..b7f93b7 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh
@@ -39,6 +39,8 @@ execute_robot_test ${SCM} s3
execute_robot_test ${SCM} admincli
+execute_robot_test ${SCM} -v SECURITY_ENABLED:${SECURITY_ENABLED} httpfs
+
export SCM=scm2.org
execute_robot_test ${SCM} admincli
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
index cfd0de8..1a23273 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -63,7 +63,22 @@ services:
env_file:
- docker-config
command: ["/opt/hadoop/bin/ozone","om"]
-
+ httpfs:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: httpfs
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ ports:
+ - 14000:14000
+ env_file:
+ - ./docker-config
+ command: [ "/opt/hadoop/bin/ozone","httpfs" ]
+ environment:
+ OZONE_CLASSPATH: "/opt/hadoop/share/ozone/lib/ozone-filesystem-1.2.0-SNAPSHOT.jar:/opt/hadoop/share/ozone/lib/ozone-filesystem-common-1.2.0-SNAPSHOT.jar:/opt/hadoop/share/ozone/lib/ozone-filesystem-hadoop3-1.2.0-SNAPSHOT.jar"
+ OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1}
+ OZONE_OPTS:
s3g:
image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
hostname: s3g
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
index 2a109fd..35c0c3c 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -16,6 +16,9 @@
CORE-SITE.XML_fs.defaultFS=ofs://om
CORE-SITE.XML_fs.trash.interval=1
+# For HttpFS it is required to enable impersonating users as root, because the httpfs is mapped to root as a result of the auth_to_local rules.
+CORE-SITE.XML_hadoop.proxyuser.root.hosts=*
+CORE-SITE.XML_hadoop.proxyuser.root.groups=*
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
@@ -44,7 +47,7 @@ OZONE-SITE.XML_ozone.recon.address=recon:9891
OZONE-SITE.XML_ozone.security.enabled=true
OZONE-SITE.XML_ozone.acl.enabled=true
OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer
-OZONE-SITE.XML_ozone.administrators="testuser/scm@EXAMPLE.COM,testuser/s3g@EXAMPLE.COM,recon/recon@EXAMPLE.COM"
+OZONE-SITE.XML_ozone.administrators="testuser/scm@EXAMPLE.COM,testuser/s3g@EXAMPLE.COM,testuser/httpfs@EXAMPLE.COM,recon/recon@EXAMPLE.COM"
OZONE-SITE.XML_ozone.recon.administrators="testuser2/scm@EXAMPLE.COM"
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
@@ -66,6 +69,9 @@ OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM
OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM
+OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab
+OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM
+
OZONE-SITE.XML_hdds.scm.replication.thread.interval=5s
OZONE-SITE.XML_hdds.scm.replication.event.timeout=10s
OZONE-SITE.XML_ozone.scm.stale.node.interval=30s
@@ -85,6 +91,7 @@ OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos
OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos
OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
@@ -95,6 +102,8 @@ OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/dn@EXAMPLE.COM
OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/dn.keytab
OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.keytab
OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
+OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
+OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM
OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=HTTP/recon@EXAMPLE.COM
OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab
@@ -115,6 +124,13 @@ HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+HTTPFS-SITE.XML_hadoop.http.authentication.type=kerberos
+HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
+HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/httpfs@EXAMPLE.COM
+HTTPFS-SITE.XML_httpfs.hadoop.authentication.type=kerberos
+HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
+HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal=httpfs/httpfs@EXAMPLE.COM
+
#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
index 5477a76..cd6160a 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
@@ -54,6 +54,8 @@ execute_robot_test scm recon
execute_robot_test scm admincli
execute_robot_test scm spnego
+execute_robot_test scm -v SECURITY_ENABLED:${SECURITY_ENABLED} httpfs
+
# test replication
docker-compose up -d --scale datanode=2
execute_robot_test scm -v container:1 -v count:2 replication/wait.robot
diff --git a/hadoop-ozone/dist/src/main/keytabs/HTTP.keytab b/hadoop-ozone/dist/src/main/keytabs/HTTP.keytab
index 9e7a369..dec34bd 100755
Binary files a/hadoop-ozone/dist/src/main/keytabs/HTTP.keytab and b/hadoop-ozone/dist/src/main/keytabs/HTTP.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/dn.keytab b/hadoop-ozone/dist/src/main/keytabs/dn.keytab
index ee982cb..657da10 100755
Binary files a/hadoop-ozone/dist/src/main/keytabs/dn.keytab and b/hadoop-ozone/dist/src/main/keytabs/dn.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/hadoop.keytab b/hadoop-ozone/dist/src/main/keytabs/hadoop.keytab
index 68f5238..40be760 100755
Binary files a/hadoop-ozone/dist/src/main/keytabs/hadoop.keytab and b/hadoop-ozone/dist/src/main/keytabs/hadoop.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/httpfs.keytab b/hadoop-ozone/dist/src/main/keytabs/httpfs.keytab
new file mode 100755
index 0000000..cfbbe68
Binary files /dev/null and b/hadoop-ozone/dist/src/main/keytabs/httpfs.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/jhs.keytab b/hadoop-ozone/dist/src/main/keytabs/jhs.keytab
index 51eeebf..e570dc6 100755
Binary files a/hadoop-ozone/dist/src/main/keytabs/jhs.keytab and b/hadoop-ozone/dist/src/main/keytabs/jhs.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/nm.keytab b/hadoop-ozone/dist/src/main/keytabs/nm.keytab
index 5cfcf9a..b582fe9 100755
Binary files a/hadoop-ozone/dist/src/main/keytabs/nm.keytab and b/hadoop-ozone/dist/src/main/keytabs/nm.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/om.keytab b/hadoop-ozone/dist/src/main/keytabs/om.keytab
index ec571b3..adffa63 100755
Binary files a/hadoop-ozone/dist/src/main/keytabs/om.keytab and b/hadoop-ozone/dist/src/main/keytabs/om.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/recon.keytab b/hadoop-ozone/dist/src/main/keytabs/recon.keytab
index 653e075..d994028 100755
Binary files a/hadoop-ozone/dist/src/main/keytabs/recon.keytab and b/hadoop-ozone/dist/src/main/keytabs/recon.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/rm.keytab b/hadoop-ozone/dist/src/main/keytabs/rm.keytab
index d9a55f0..9560944 100755
Binary files a/hadoop-ozone/dist/src/main/keytabs/rm.keytab and b/hadoop-ozone/dist/src/main/keytabs/rm.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/s3g.keytab b/hadoop-ozone/dist/src/main/keytabs/s3g.keytab
index be66846..a89b337 100755
Binary files a/hadoop-ozone/dist/src/main/keytabs/s3g.keytab and b/hadoop-ozone/dist/src/main/keytabs/s3g.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/scm.keytab b/hadoop-ozone/dist/src/main/keytabs/scm.keytab
index dbd3166..eb7d81f 100755
Binary files a/hadoop-ozone/dist/src/main/keytabs/scm.keytab and b/hadoop-ozone/dist/src/main/keytabs/scm.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/testuser.keytab b/hadoop-ozone/dist/src/main/keytabs/testuser.keytab
index c9c4550..c9aa8b8 100755
Binary files a/hadoop-ozone/dist/src/main/keytabs/testuser.keytab and b/hadoop-ozone/dist/src/main/keytabs/testuser.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/testuser2.keytab b/hadoop-ozone/dist/src/main/keytabs/testuser2.keytab
index 57c77ca..91cfb3e 100755
Binary files a/hadoop-ozone/dist/src/main/keytabs/testuser2.keytab and b/hadoop-ozone/dist/src/main/keytabs/testuser2.keytab differ
diff --git a/hadoop-ozone/dist/src/main/keytabs/update-keytabs.sh b/hadoop-ozone/dist/src/main/keytabs/update-keytabs.sh
index be60da6..b55923e 100755
--- a/hadoop-ozone/dist/src/main/keytabs/update-keytabs.sh
+++ b/hadoop-ozone/dist/src/main/keytabs/update-keytabs.sh
@@ -36,6 +36,7 @@ if [ "$1" == "internal" ]; then
export_keytab testuser/scm testuser
export_keytab testuser/s3g testuser
+ export_keytab testuser/httpfs testuser
export_keytab testuser/om testuser
export_keytab testuser2/scm testuser2
@@ -47,6 +48,10 @@ if [ "$1" == "internal" ]; then
export_keytab HTTP/s3g s3g
export_keytab testuser/s3g s3g
+ export_keytab httpfs/httpfs httpfs
+ export_keytab HTTP/httpfs httpfs
+ export_keytab testuser/httpfs httpfs
+
export_keytab recon/recon recon
export_keytab HTTP/recon recon
@@ -55,6 +60,7 @@ if [ "$1" == "internal" ]; then
export_keytab HTTP/scm HTTP
export_keytab HTTP/s3g HTTP
+ export_keytab HTTP/httpfs HTTP
export_keytab HTTP/ozone HTTP
export_keytab hadoop/rm hadoop
diff --git a/hadoop-ozone/dist/src/main/smoketest/httpfs/operations.robot b/hadoop-ozone/dist/src/main/smoketest/httpfs/operations.robot
index 03658b9..ac9b56f 100644
--- a/hadoop-ozone/dist/src/main/smoketest/httpfs/operations.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/httpfs/operations.robot
@@ -19,24 +19,27 @@ Library BuiltIn
Library String
*** Variables ***
-${URL} http://httpfs:14000/webhdfs/v1/
-${USERNAME} hdfs
+${URL} http://httpfs:14000/webhdfs/v1/
*** Keywords ***
Execute curl command
[Arguments] ${path} ${operation} ${extra_commands}
- ${final_url} = Catenate SEPARATOR= ${URL} ${path} ?op= ${operation} &user.name= ${USERNAME}
- ${output} Run process curl ${extra_commands} "${final_url}" shell=True
+ ${user.name} = Set Variable If '${SECURITY_ENABLED}'=='false' &user.name=${USERNAME} ${EMPTY}
+ ${final_url} = Catenate SEPARATOR= ${URL} ${path} ?op= ${operation} ${user.name}
+ ${curl_extra_commands} = Set Variable If '${SECURITY_ENABLED}'=='true' --negotiate -u : ${EMPTY}
+ ${output} Run process curl ${extra_commands} ${curl_extra_commands} "${final_url}" shell=True
Should Be Equal As Integers ${output.rc} 0
[return] ${output}
Execute create file command
[Arguments] ${path} ${file_name}
- ${final_url} = Catenate SEPARATOR= ${URL} ${path} ?op=CREATE&user.name= ${USERNAME}
- ${output} Run process curl -X PUT "${final_url}" shell=True
+ ${user.name} = Set Variable If '${SECURITY_ENABLED}'=='false' &user.name=${USERNAME} ${EMPTY}
+ ${curl_extra_commands} = Set Variable If '${SECURITY_ENABLED}'=='true' --negotiate -u : ${EMPTY}
+ ${final_url} = Catenate SEPARATOR= ${URL} ${path} ?op=CREATE ${user.name}
+ ${output} Run process curl -X PUT ${curl_extra_commands} "${final_url}" shell=True
Should Be Equal As Integers ${output.rc} 0
- ${final_url2} = Catenate SEPARATOR= ${URL} ${path} ?op=CREATE&data=true&user.name= ${USERNAME}
- ${output2} Run process curl -X PUT -T ${file_name} "${final_url2}" -H"Content-Type: application/octet-stream" shell=True
+ ${final_url2} = Catenate SEPARATOR= ${URL} ${path} ?op=CREATE&data=true ${user.name}
+ ${output2} Run process curl -X PUT -T ${file_name} ${curl_extra_commands} "${final_url2}" -H"Content-Type: application/octet-stream" shell=True
Should Be Equal As Integers ${output2.rc} 0
[return] ${output2}
diff --git a/hadoop-ozone/dist/src/main/smoketest/httpfs/operations_tests.robot b/hadoop-ozone/dist/src/main/smoketest/httpfs/operations_tests.robot
index be19e3b..758630f 100644
--- a/hadoop-ozone/dist/src/main/smoketest/httpfs/operations_tests.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/httpfs/operations_tests.robot
@@ -19,10 +19,11 @@ Library Process
Library String
Library BuiltIn
Resource operations.robot
+Resource ../lib/os.robot
+Resource ../commonlib.robot
+Suite Setup Generate volume
*** Variables ***
-${URL} http://httpfs:14000/webhdfs/v1/
-${USERNAME} hdfs
${volume} generated
*** Keywords ***
@@ -30,10 +31,26 @@ Generate volume
${random} = Generate Random String 5 [LOWER]
Set Suite Variable ${volume} ${random}
+Kinit admin
+ Wait Until Keyword Succeeds 2min 10sec Execute kinit -k httpfs/httpfs@EXAMPLE.COM -t /etc/security/keytabs/httpfs.keytab
+
*** Test Cases ***
+Kinit admin user
+ Pass Execution If '${SECURITY_ENABLED}'=='false' This is for secured environment
+ Kinit admin
+
Create volume
- ${volume} = Execute curl command ${volume} MKDIRS -X PUT
- Should contain ${volume.stdout} true
+ ${vol} = Execute curl command ${volume} MKDIRS -X PUT
+ Should contain ${vol.stdout} true
+
+Set owner of volume
+ Pass Execution If '${SECURITY_ENABLED}'=='false' This is for secured environment
+ ${rc} = Run And Return Rc ozone sh volume update --user=testuser /${volume}
+ Should Be Equal As Integers ${rc} 0
+
+Kinit testuser
+ Pass Execution If '${SECURITY_ENABLED}'=='false' This is for secured environment
+ Kinit test user testuser testuser.keytab
Create first bucket
${bucket} = Execute curl command ${volume}/buck1 MKDIRS -X PUT
@@ -90,11 +107,13 @@ Get quota usage of directory
Get home directory
${home} = Execute curl command ${EMPTY} GETHOMEDIRECTORY ${EMPTY}
- Should contain ${home.stdout} "Path":"\\/user\\/hdfs"
+ ${user} = Set Variable If '${SECURITY_ENABLED}'=='true' testuser ${USERNAME}
+ Should contain ${home.stdout} "Path":"\\/user\\/${user}"
Get trash root
${trash} = Execute curl command ${volume}/buck1/testfile GETTRASHROOT ${EMPTY}
- Should contain ${trash.stdout} "Path":"\\/${volume}\\/buck1\\/.Trash\\/hdfs"
+ ${user} = Set Variable If '${SECURITY_ENABLED}'=='true' testuser ${USERNAME}
+ Should contain ${trash.stdout} "Path":"\\/${volume}\\/buck1\\/.Trash\\/${user}"
# Missing functionality, not working yet.
# Set permission of bucket
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org