You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by GitBox <gi...@apache.org> on 2018/07/26 11:21:18 UTC
[GitHub] aljoscha closed pull request #6377: [FLINK-8981] Add end-to-end
test for running on YARN with Kerberos
aljoscha closed pull request #6377: [FLINK-8981] Add end-to-end test for running on YARN with Kerberos
URL: https://github.com/apache/flink/pull/6377
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index dc8424f25ec..db75ae89324 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -18,7 +18,7 @@
################################################################################
END_TO_END_DIR="`dirname \"$0\"`" # relative
-END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd )`" # absolutized and normalized
+END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd -P)`" # absolutized and normalized
if [ -z "$END_TO_END_DIR" ] ; then
# error; for some reason, the path is not accessible
# to the script (e.g. permissions re-evaled after suid)
@@ -34,7 +34,7 @@ fi
source "${END_TO_END_DIR}/test-scripts/test-runner-common.sh"
-FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd )`" # absolutized and normalized
+FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd -P)`" # absolutized and normalized
echo "flink-end-to-end-test directory: $END_TO_END_DIR"
echo "Flink distribution directory: $FLINK_DIR"
@@ -105,5 +105,7 @@ run_test "Avro Confluent Schema Registry nightly end-to-end test" "$END_TO_END_D
run_test "State TTL Heap backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh file"
run_test "State TTL RocksDb backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh rocks"
+run_test "Running Kerberized YARN on Docker test " "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh"
+
printf "\n[PASS] All tests passed\n"
exit 0
diff --git a/flink-end-to-end-tests/run-pre-commit-tests.sh b/flink-end-to-end-tests/run-pre-commit-tests.sh
index 6355fd0f2aa..7b9777c43f3 100755
--- a/flink-end-to-end-tests/run-pre-commit-tests.sh
+++ b/flink-end-to-end-tests/run-pre-commit-tests.sh
@@ -18,7 +18,7 @@
################################################################################
END_TO_END_DIR="`dirname \"$0\"`" # relative
-END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd )`" # absolutized and normalized
+END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd -P )`" # absolutized and normalized
if [ -z "$END_TO_END_DIR" ] ; then
# error; for some reason, the path is not accessible
# to the script (e.g. permissions re-evaled after suid)
@@ -34,7 +34,7 @@ fi
source ${END_TO_END_DIR}/test-scripts/test-runner-common.sh
-FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd )`" # absolutized and normalized
+FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd -P)`" # absolutized and normalized
echo "flink-end-to-end-test directory: $END_TO_END_DIR"
echo "Flink distribution directory: $FLINK_DIR"
diff --git a/flink-end-to-end-tests/run-single-test.sh b/flink-end-to-end-tests/run-single-test.sh
index 86b313d757f..833a78a5af3 100755
--- a/flink-end-to-end-tests/run-single-test.sh
+++ b/flink-end-to-end-tests/run-single-test.sh
@@ -26,7 +26,7 @@ if [ $# -eq 0 ]; then
fi
END_TO_END_DIR="`dirname \"$0\"`" # relative
-END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd )`" # absolutized and normalized
+END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd -P)`" # absolutized and normalized
if [ -z "$END_TO_END_DIR" ] ; then
# error; for some reason, the path is not accessible
# to the script (e.g. permissions re-evaled after suid)
@@ -42,7 +42,7 @@ fi
source "${END_TO_END_DIR}/test-scripts/test-runner-common.sh"
-FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd )`" # absolutized and normalized
+FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd -P )`" # absolutized and normalized
echo "flink-end-to-end-test directory: $END_TO_END_DIR"
echo "Flink distribution directory: $FLINK_DIR"
diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh
index f4563cc3ea4..621db11e824 100644
--- a/flink-end-to-end-tests/test-scripts/common.sh
+++ b/flink-end-to-end-tests/test-scripts/common.sh
@@ -37,10 +37,10 @@ export EXIT_CODE=0
echo "Flink dist directory: $FLINK_DIR"
USE_SSL=OFF # set via set_conf_ssl(), reset via revert_default_config()
-TEST_ROOT=`pwd`
+TEST_ROOT=`pwd -P`
TEST_INFRA_DIR="$END_TO_END_DIR/test-scripts/"
cd $TEST_INFRA_DIR
-TEST_INFRA_DIR=`pwd`
+TEST_INFRA_DIR=`pwd -P`
cd $TEST_ROOT
function print_mem_use_osx {
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/Dockerfile b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/Dockerfile
new file mode 100644
index 00000000000..2ec94dccc7b
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/Dockerfile
@@ -0,0 +1,162 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+#
+# This image is modified version of Knappek/docker-hadoop-secure
+# * Knappek/docker-hadoop-secure <https://github.com/Knappek/docker-hadoop-secure>
+#
+# With bits and pieces added from Lewuathe/docker-hadoop-cluster to extend it to start a proper kerberized Hadoop cluster:
+# * Lewuathe/docker-hadoop-cluster <https://github.com/Lewuathe/docker-hadoop-cluster>
+#
+# Author: Aljoscha Krettek
+# Date: 2018 May, 15
+#
+# Creates multi-node, kerberized Hadoop cluster on Docker
+
+FROM sequenceiq/pam:ubuntu-14.04
+MAINTAINER aljoscha
+
+RUN set -x \
+ && addgroup hadoop \
+ && useradd -d /home/hdfs -ms /bin/bash -G hadoop -p hdfs hdfs \
+ && useradd -d /home/yarn -ms /bin/bash -G hadoop -p yarn yarn \
+ && useradd -d /home/mapred -ms /bin/bash -G hadoop -p mapred mapred \
+ && useradd -d /home/hadoop-user -ms /bin/bash -p hadoop-user hadoop-user
+
+# install dev tools
+RUN set -x \
+ && apt-get update && apt-get install -y \
+ curl tar sudo openssh-server openssh-client rsync unzip krb5-user
+
+# Kerberos client
+RUN set -x \
+ && mkdir -p /var/log/kerberos \
+ && touch /var/log/kerberos/kadmind.log
+
+# passwordless ssh
+RUN set -x \
+ && rm -f /etc/ssh/ssh_host_dsa_key /etc/ssh/ssh_host_rsa_key /root/.ssh/id_rsa \
+ && ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key \
+ && ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key \
+ && ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa \
+ && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+
+# java
+RUN set -x \
+ && mkdir -p /usr/java/default \
+ && curl -Ls 'http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz' -H 'Cookie: oraclelicense=accept-securebackup-cookie' | \
+ tar --strip-components=1 -xz -C /usr/java/default/
+
+ENV JAVA_HOME /usr/java/default
+ENV PATH $PATH:$JAVA_HOME/bin
+
+RUN set -x \
+ && curl -LOH 'Cookie: oraclelicense=accept-securebackup-cookie' 'http://download.oracle.com/otn-pub/java/jce/8/jce_policy-8.zip' \
+ && unzip jce_policy-8.zip \
+ && cp /UnlimitedJCEPolicyJDK8/local_policy.jar /UnlimitedJCEPolicyJDK8/US_export_policy.jar $JAVA_HOME/jre/lib/security
+
+ARG HADOOP_VERSION=2.8.4
+
+ENV HADOOP_URL http://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
+RUN set -x \
+ && curl -fSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz \
+ && tar -xf /tmp/hadoop.tar.gz -C /usr/local/ \
+ && rm /tmp/hadoop.tar.gz*
+
+WORKDIR /usr/local
+RUN set -x \
+ && ln -s /usr/local/hadoop-${HADOOP_VERSION} /usr/local/hadoop \
+ && chown root:root -R /usr/local/hadoop-${HADOOP_VERSION}/ \
+ && chown root:root -R /usr/local/hadoop/ \
+ && chown root:yarn /usr/local/hadoop/bin/container-executor \
+ && chmod 6050 /usr/local/hadoop/bin/container-executor \
+ && mkdir -p /hadoop-data/nm-local-dirs \
+ && mkdir -p /hadoop-data/nm-log-dirs \
+ && chown yarn:yarn /hadoop-data \
+ && chown yarn:yarn /hadoop-data/nm-local-dirs \
+ && chown yarn:yarn /hadoop-data/nm-log-dirs \
+ && chmod 755 /hadoop-data \
+ && chmod 755 /hadoop-data/nm-local-dirs \
+ && chmod 755 /hadoop-data/nm-log-dirs
+
+ENV HADOOP_HOME /usr/local/hadoop
+ENV HADOOP_COMMON_HOME /usr/local/hadoop
+ENV HADOOP_HDFS_HOME /usr/local/hadoop
+ENV HADOOP_MAPRED_HOME /usr/local/hadoop
+ENV HADOOP_YARN_HOME /usr/local/hadoop
+ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop
+ENV YARN_CONF_DIR /usr/local/hadoop/etc/hadoop
+ENV HADOOP_LOG_DIR /var/log/hadoop
+ENV HADOOP_BIN_HOME $HADOOP_HOME/bin
+ENV PATH $PATH:$HADOOP_BIN_HOME
+
+ENV KRB_REALM EXAMPLE.COM
+ENV DOMAIN_REALM example.com
+ENV KERBEROS_ADMIN admin/admin
+ENV KERBEROS_ADMIN_PASSWORD admin
+ENV KEYTAB_DIR /etc/security/keytabs
+
+RUN mkdir /var/log/hadoop
+
+ADD config/core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml
+ADD config/hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml
+ADD config/mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml
+ADD config/yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml
+ADD config/container-executor.cfg $HADOOP_HOME/etc/hadoop/container-executor.cfg
+ADD config/krb5.conf /etc/krb5.conf
+ADD config/ssl-server.xml $HADOOP_HOME/etc/hadoop/ssl-server.xml
+ADD config/ssl-client.xml $HADOOP_HOME/etc/hadoop/ssl-client.xml
+ADD config/keystore.jks $HADOOP_HOME/lib/keystore.jks
+
+RUN set -x \
+ && chmod 400 $HADOOP_HOME/etc/hadoop/container-executor.cfg \
+ && chown root:yarn $HADOOP_HOME/etc/hadoop/container-executor.cfg
+
+ADD config/ssh_config /root/.ssh/config
+RUN set -x \
+ && chmod 600 /root/.ssh/config \
+ && chown root:root /root/.ssh/config
+
+# workingaround docker.io build error
+RUN set -x \
+ && ls -la /usr/local/hadoop/etc/hadoop/*-env.sh \
+ && chmod +x /usr/local/hadoop/etc/hadoop/*-env.sh \
+ && ls -la /usr/local/hadoop/etc/hadoop/*-env.sh
+
+# fix the 254 error code
+RUN set -x \
+ && sed -i "/^[^#]*UsePAM/ s/.*/#&/" /etc/ssh/sshd_config \
+ && echo "UsePAM no" >> /etc/ssh/sshd_config \
+ && echo "Port 2122" >> /etc/ssh/sshd_config
+
+# Hdfs ports
+EXPOSE 50470 9000 50010 50020 50070 50075 50090 50475 50091 8020
+# Mapred ports
+EXPOSE 19888
+# Yarn ports
+EXPOSE 8030 8031 8032 8033 8040 8042 8088 8188
+# Other ports
+EXPOSE 49707 2122
+
+ADD bootstrap.sh /etc/bootstrap.sh
+RUN chown root:root /etc/bootstrap.sh
+RUN chmod 700 /etc/bootstrap.sh
+
+ENV BOOTSTRAP /etc/bootstrap.sh
+
+ENTRYPOINT ["/etc/bootstrap.sh"]
+CMD ["-h"]
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/README.md b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/README.md
new file mode 100644
index 00000000000..a9f96096be2
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/README.md
@@ -0,0 +1,118 @@
+# Apache Hadoop Docker image with Kerberos enabled
+
+This image is modified version of Knappek/docker-hadoop-secure
+ * Knappek/docker-hadoop-secure <https://github.com/Knappek/docker-hadoop-secure>
+
+With bits and pieces added from Lewuathe/docker-hadoop-cluster to extend it to start a proper kerberized Hadoop cluster:
+ * Lewuathe/docker-hadoop-cluster <https://github.com/Lewuathe/docker-hadoop-cluster>
+
+And a lot of added stuff for making this an actual, properly configured, kerberized cluster with proper user/permissions structure.
+
+Versions
+--------
+
+* JDK8
+* Hadoop 2.8.3
+
+Default Environment Variables
+-----------------------------
+
+| Name | Value | Description |
+| ---- | ---- | ---- |
+| `KRB_REALM` | `EXAMPLE.COM` | The Kerberos Realm, more information [here](https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html#) |
+| `DOMAIN_REALM` | `example.com` | The Kerberos Domain Realm, more information [here](https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html#) |
+| `KERBEROS_ADMIN` | `admin/admin` | The KDC admin user |
+| `KERBEROS_ADMIN_PASSWORD` | `admin` | The KDC admin password |
+
+You can simply define these variables in the `docker-compose.yml`.
+
+Run image
+---------
+
+Clone the [project](https://git-wip-us.apache.org/repos/asf/flink.git) and run
+
+```
+docker-compose up
+```
+
+Usage
+-----
+
+Get the container name with `docker ps` and login to the container with
+
+```
+docker exec -it <container-name> /bin/bash
+```
+
+
+To obtain a Kerberos ticket, execute
+
+```
+kinit -kt /home/hadoop-user/hadoop-user.keytab hadoop-user
+```
+
+Afterwards you can use `hdfs` CLI like
+
+```
+hdfs dfs -ls /
+```
+
+
+Known issues
+------------
+
+### Unable to obtain Kerberos password
+
+#### Error
+docker-compose up fails for the first time with the error
+
+```
+Login failure for nn/hadoop.docker.com@EXAMPLE.COM from keytab /etc/security/keytabs/nn.service.keytab: javax.security.auth.login.LoginException: Unable to obtain password from user
+```
+
+#### Solution
+
+Stop the containers with `docker-compose down` and start again with `docker-compose up -d`.
+
+
+### JDK 8
+
+Make sure you use download a JDK version that is still available. Old versions can be deprecated by Oracle and thus the download link won't be able anymore.
+
+Get the latest JDK8 Download URL with
+
+```
+curl -s https://lv.binarybabel.org/catalog-api/java/jdk8.json
+```
+
+### Java Keystore
+
+If the Keystore has been expired, then create a new `keystore.jks`:
+
+1. create private key
+
+```
+openssl genrsa -des3 -out server.key 1024
+```
+
+2. create csr
+
+```
+openssl req -new -key server.key -out server.csr`
+```
+
+3. remove passphrase in key
+```
+cp server.key server.key.org
+openssl rsa -in server.key.org -out server.key
+```
+
+3. create self-signed cert
+```
+openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt
+```
+
+4. create JKS and import certificate
+```
+keytool -import -keystore keystore.jks -alias CARoot -file server.crt`
+```
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/bootstrap.sh b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/bootstrap.sh
new file mode 100755
index 00000000000..6f43182b106
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/bootstrap.sh
@@ -0,0 +1,119 @@
+#!/bin/bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+
+$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+
+rm /tmp/*.pid
+
+# installing libraries if any - (resource urls added comma separated to the ACP system variable)
+cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd -
+
+# kerberos client
+sed -i "s/EXAMPLE.COM/${KRB_REALM}/g" /etc/krb5.conf
+sed -i "s/example.com/${DOMAIN_REALM}/g" /etc/krb5.conf
+
+# update config files
+sed -i "s/HOSTNAME/$(hostname -f)/g" $HADOOP_PREFIX/etc/hadoop/core-site.xml
+sed -i "s/EXAMPLE.COM/${KRB_REALM}/g" $HADOOP_PREFIX/etc/hadoop/core-site.xml
+sed -i "s#/etc/security/keytabs#${KEYTAB_DIR}#g" $HADOOP_PREFIX/etc/hadoop/core-site.xml
+
+sed -i "s/EXAMPLE.COM/${KRB_REALM}/g" $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
+sed -i "s/HOSTNAME/$(hostname -f)/g" $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
+sed -i "s#/etc/security/keytabs#${KEYTAB_DIR}#g" $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
+
+sed -i "s/EXAMPLE.COM/${KRB_REALM}/g" $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
+sed -i "s/HOSTNAME/$(hostname -f)/g" $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
+sed -i "s#/etc/security/keytabs#${KEYTAB_DIR}#g" $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
+
+sed -i "s/EXAMPLE.COM/${KRB_REALM}/g" $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
+sed -i "s/HOSTNAME/$(hostname -f)/g" $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
+sed -i "s#/etc/security/keytabs#${KEYTAB_DIR}#g" $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
+
+sed -i "s#/usr/local/hadoop/bin/container-executor#${NM_CONTAINER_EXECUTOR_PATH}#g" $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
+
+# create namenode kerberos principal and keytab
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "addprinc -randkey hdfs/$(hostname -f)@${KRB_REALM}"
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "addprinc -randkey mapred/$(hostname -f)@${KRB_REALM}"
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "addprinc -randkey yarn/$(hostname -f)@${KRB_REALM}"
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "addprinc -randkey HTTP/$(hostname -f)@${KRB_REALM}"
+
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "xst -k hdfs.keytab hdfs/$(hostname -f) HTTP/$(hostname -f)"
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "xst -k mapred.keytab mapred/$(hostname -f) HTTP/$(hostname -f)"
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "xst -k yarn.keytab yarn/$(hostname -f) HTTP/$(hostname -f)"
+
+mkdir -p ${KEYTAB_DIR}
+mv hdfs.keytab ${KEYTAB_DIR}
+mv mapred.keytab ${KEYTAB_DIR}
+mv yarn.keytab ${KEYTAB_DIR}
+chmod 400 ${KEYTAB_DIR}/hdfs.keytab
+chmod 400 ${KEYTAB_DIR}/mapred.keytab
+chmod 400 ${KEYTAB_DIR}/yarn.keytab
+chown hdfs:hadoop ${KEYTAB_DIR}/hdfs.keytab
+chown mapred:hadoop ${KEYTAB_DIR}/mapred.keytab
+chown yarn:hadoop ${KEYTAB_DIR}/yarn.keytab
+
+service ssh start
+
+if [ "$1" == "--help" -o "$1" == "-h" ]; then
+ echo "Usage: $(basename $0) (master|worker)"
+ exit 0
+elif [ "$1" == "master" ]; then
+ yes| sudo -E -u hdfs $HADOOP_PREFIX/bin/hdfs namenode -format
+
+ nohup sudo -E -u hdfs $HADOOP_PREFIX/bin/hdfs namenode 2>> /var/log/hadoop/namenode.err >> /var/log/hadoop/namenode.out &
+ nohup sudo -E -u yarn $HADOOP_PREFIX/bin/yarn resourcemanager 2>> /var/log/hadoop/resourcemanager.err >> /var/log/hadoop/resourcemanager.out &
+ nohup sudo -E -u yarn $HADOOP_PREFIX/bin/yarn timelineserver 2>> /var/log/hadoop/timelineserver.err >> /var/log/hadoop/timelineserver.out &
+ nohup sudo -E -u mapred $HADOOP_PREFIX/bin/mapred historyserver 2>> /var/log/hadoop/historyserver.err >> /var/log/hadoop/historyserver.out &
+
+
+ kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "addprinc -randkey root@${KRB_REALM}"
+ kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "xst -k /root/root.keytab root"
+
+ kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "addprinc -pw hadoop-user hadoop-user@${KRB_REALM}"
+ kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "xst -k /home/hadoop-user/hadoop-user.keytab hadoop-user"
+ chown hadoop-user:hadoop-user /home/hadoop-user/hadoop-user
+
+ kinit -kt /root/root.keytab root
+
+ hdfs dfsadmin -safemode wait
+ while [ $? -ne 0 ]; do hdfs dfsadmin -safemode wait; done
+
+ hdfs dfs -chown hdfs:hadoop /
+ hdfs dfs -chmod 755 /
+ hdfs dfs -mkdir /tmp
+ hdfs dfs -chown hdfs:hadoop /tmp
+ hdfs dfs -chmod -R 1777 /tmp
+ hdfs dfs -mkdir /tmp/logs
+ hdfs dfs -chown yarn:hadoop /tmp/logs
+ hdfs dfs -chmod 1777 /tmp/logs
+
+ hdfs dfs -mkdir -p /user/hadoop-user
+ hdfs dfs -chown hadoop-user:hadoop-user /user/hadoop-user
+
+ kdestroy
+
+ while true; do sleep 1000; done
+elif [ "$1" == "worker" ]; then
+ nohup sudo -E -u hdfs $HADOOP_PREFIX/bin/hdfs datanode 2>> /var/log/hadoop/datanode.err >> /var/log/hadoop/datanode.out &
+ nohup sudo -E -u yarn $HADOOP_PREFIX/bin/yarn nodemanager 2>> /var/log/hadoop/nodemanager.err >> /var/log/hadoop/nodemanager.out &
+ while true; do sleep 1000; done
+fi
+
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/container-executor.cfg b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/container-executor.cfg
new file mode 100644
index 00000000000..e9de3478955
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/container-executor.cfg
@@ -0,0 +1,23 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+yarn.nodemanager.linux-container-executor.group=yarn
+yarn.nodemanager.local-dirs=/hadoop-data/nm-local-dirs
+yarn.nodemanager.log-dirs=/hadoop-data/nm-log-dirs
+banned.users=hdfs,yarn,mapred,bin
+min.user.id=500
\ No newline at end of file
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/core-site.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/core-site.xml
new file mode 100644
index 00000000000..366d13037f5
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/core-site.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://master.docker-hadoop-cluster-network:9000</value>
+ </property>
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ <description> Set the authentication for the cluster.
+ Valid values are: simple or kerberos.</description>
+ </property>
+ <property>
+ <name>hadoop.security.authorization</name>
+ <value>true</value>
+ <description>Enable authorization for different protocols.</description>
+ </property>
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value>
+ RULE:[2:$1](.*)
+ DEFAULT
+ </value>
+ <description>The mapping from kerberos principal names
+ to local OS user names.</description>
+ </property>
+ <property>
+ <name>hadoop.ssl.require.client.cert</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hadoop.ssl.hostname.verifier</name>
+ <value>DEFAULT</value>
+ </property>
+ <property>
+ <name>hadoop.ssl.keystores.factory.class</name>
+ <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
+ </property>
+ <property>
+ <name>hadoop.ssl.server.conf</name>
+ <value>ssl-server.xml</value>
+ </property>
+ <property>
+ <name>hadoop.ssl.client.conf</name>
+ <value>ssl-client.xml</value>
+ </property>
+ <property>
+ <name>hadoop.rpc.protection</name>
+ <value>privacy</value>
+ </property>
+</configuration>
+
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/hdfs-site.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/hdfs-site.xml
new file mode 100644
index 00000000000..a4e5ed8876a
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/hdfs-site.xml
@@ -0,0 +1,190 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<configuration>
+ <property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>dfs.namenode.rpc-address</name>
+ <value>master.docker-hadoop-cluster-network:9000</value>
+ </property>
+ <property>
+ <name>dfs.permissions</name>
+ <value>true</value>
+ <description> If "true", enable permission checking in
+ HDFS. If "false", permission checking is turned
+ off, but all other behavior is
+ unchanged. Switching from one parameter value to the other does
+ not change the mode, owner or group of files or
+ directories. </description>
+ </property>
+
+ <property>
+ <name>dfs.permissions.supergroup</name>
+ <value>root</value>
+ <description>The name of the group of super-users.</description>
+ </property>
+
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>100</value>
+ <description>Added to grow Queue size so that more client connections are allowed</description>
+ </property>
+
+ <property>
+ <name>ipc.server.max.response.size</name>
+ <value>5242880</value>
+ </property>
+
+ <property>
+ <name>dfs.block.access.token.enable</name>
+ <value>true</value>
+ <description> If "true", access tokens are used as capabilities
+ for accessing datanodes. If "false", no access tokens are checked on
+ accessing datanodes. </description>
+ </property>
+
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value>hdfs/_HOST@EXAMPLE.COM</value>
+ <description> Kerberos principal name for the NameNode </description>
+ </property>
+
+ <property>
+ <name>dfs.secondary.namenode.kerberos.principal</name>
+ <value>hdfs/_HOST@EXAMPLE.COM</value>
+ <description>Kerberos principal name for the secondary NameNode.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.secondary.http.address</name>
+ <value>master:50090</value>
+ <description>Address of secondary namenode web server</description>
+ </property>
+
+ <property>
+ <name>dfs.secondary.https.port</name>
+ <value>50490</value>
+ <description>The https port where secondary-namenode binds</description>
+ </property>
+
+ <property>
+ <name>dfs.web.authentication.kerberos.principal</name>
+ <value>HTTP/_HOST@EXAMPLE.COM</value>
+ <description> The HTTP Kerberos principal used by Hadoop-Auth in the HTTP endpoint.
+ The HTTP Kerberos principal MUST start with 'HTTP/' per Kerberos HTTP
+ SPNEGO specification.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.web.authentication.kerberos.keytab</name>
+ <value>/etc/security/keytabs/hdfs.keytab</value>
+ <description>The Kerberos keytab file with the credentials for the HTTP
+ Kerberos principal used by Hadoop-Auth in the HTTP endpoint.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.datanode.kerberos.principal</name>
+ <value>hdfs/_HOST@EXAMPLE.COM</value>
+ <description>
+ The Kerberos principal that the DataNode runs as. "_HOST" is replaced by the real
+ host name.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.namenode.keytab.file</name>
+ <value>/etc/security/keytabs/hdfs.keytab</value>
+ <description>
+ Combined keytab file containing the namenode service and host
+ principals.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.secondary.namenode.keytab.file</name>
+ <value>/etc/security/keytabs/hdfs.keytab</value>
+ <description>
+ Combined keytab file containing the namenode service and host
+ principals.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.datanode.keytab.file</name>
+ <value>/etc/security/keytabs/hdfs.keytab</value>
+ <description>
+ The filename of the keytab file for the DataNode.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.datanode.data.dir.perm</name>
+ <value>750</value>
+ <description>The permissions that should be there on
+ dfs.data.dir directories. The datanode will not come up if the
+ permissions are different on existing dfs.data.dir directories. If
+ the directories don't exist, they will be created with this
+ permission.</description>
+ </property>
+
+ <property>
+ <name>dfs.access.time.precision</name>
+ <value>0</value>
+ <description>The access time for HDFS file is precise upto this
+ value.The default value is 1 hour. Setting a value of 0
+ disables access times for HDFS.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cluster.administrators</name>
+ <value>root</value>
+ <description>ACL for who all can view the default servlets in the HDFS</description>
+ </property>
+
+ <property>
+ <name>ipc.server.read.threadpool.size</name>
+ <value>5</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>dfs.data.transfer.protection</name>
+ <value>authentication</value>
+ </property>
+ <property>
+ <name>dfs.encrypt.data.transfer</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.data.dir.perm</name>
+ <value>700</value>
+ </property>
+
+ <property>
+ <name>dfs.http.policy</name>
+ <value>HTTPS_ONLY</value>
+ </property>
+</configuration>
+
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/keystore.jks b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/keystore.jks
new file mode 100644
index 00000000000..31fbd2620ef
Binary files /dev/null and b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/keystore.jks differ
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/krb5.conf b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/krb5.conf
new file mode 100644
index 00000000000..a1e38c9bdc5
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/krb5.conf
@@ -0,0 +1,40 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+[logging]
+ default = FILE:/var/log/kerberos/krb5libs.log
+ kdc = FILE:/var/log/kerberos/krb5kdc.log
+ admin_server = FILE:/var/log/kerberos/kadmind.log
+
+[libdefaults]
+ default_realm = EXAMPLE.COM
+ dns_lookup_realm = false
+ dns_lookup_kdc = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+
+[realms]
+ EXAMPLE.COM = {
+ kdc = kdc
+ admin_server = kdc
+ }
+
+[domain_realm]
+ .kdc = EXAMPLE.COM
+ kdc = EXAMPLE.COM
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/mapred-site.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/mapred-site.xml
new file mode 100644
index 00000000000..fcde158f00c
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/mapred-site.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<configuration>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.keytab</name>
+ <value>/etc/security/keytabs/mapred.keytab</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.principal</name>
+ <value>mapred/HOSTNAME@EXAMPLE.COM</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>master:19888</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.webapp.https.address</name>
+ <value>master:19889</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.webapp.spnego-keytab-file</name>
+ <value>/etc/security/keytabs/mapred.keytab</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.webapp.spnego-principal</name>
+ <value>HTTP/HOSTNAME@EXAMPLE.COM</value>
+ </property>
+</configuration>
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssh_config b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssh_config
new file mode 100644
index 00000000000..486f953bf69
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssh_config
@@ -0,0 +1,23 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+Host *
+ UserKnownHostsFile /dev/null
+ StrictHostKeyChecking no
+ LogLevel quiet
+ Port 2122
\ No newline at end of file
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssl-client.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssl-client.xml
new file mode 100644
index 00000000000..b8c0085de1f
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssl-client.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<configuration>
+
+<property>
+ <name>ssl.client.truststore.location</name>
+ <value>/usr/local/hadoop/lib/keystore.jks</value>
+ <description>Truststore to be used by clients like distcp. Must be
+ specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.password</name>
+ <value>bigdata</value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload check interval, in milliseconds.
+ Default value is 10000 (10 seconds).
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.location</name>
+ <value>/usr/local/hadoop/lib/keystore.jks</value>
+ <description>Keystore to be used by clients like distcp. Must be
+ specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.password</name>
+ <value>bigdata</value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.keypassword</name>
+ <value>bigdata</value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+</configuration>
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssl-server.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssl-server.xml
new file mode 100644
index 00000000000..ce94ad7d6f7
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssl-server.xml
@@ -0,0 +1,77 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+<property>
+ <name>ssl.server.truststore.location</name>
+ <value>/usr/local/hadoop/lib/keystore.jks</value>
+ <description>Truststore to be used by NN and DN. Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.password</name>
+ <value>bigdata</value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload check interval, in milliseconds.
+ Default value is 10000 (10 seconds).
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.location</name>
+ <value>/usr/local/hadoop/lib/keystore.jks</value>
+ <description>Keystore to be used by NN and DN. Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.password</name>
+ <value>bigdata</value>
+ <description>Must be specified</description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.keypassword</name>
+ <value>bigdata</value>
+ <description>Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+</configuration>
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml
new file mode 100644
index 00000000000..62bea95301c
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml
@@ -0,0 +1,166 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<configuration>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>/usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/*</value>
+ </property>
+
+ <property>
+ <description>
+ Number of seconds after an application finishes before the nodemanager's
+ DeletionService will delete the application's localized file directory
+ and log directory.
+
+ To diagnose Yarn application problems, set this property's value large
+ enough (for example, to 600 = 10 minutes) to permit examination of these
+ directories. After changing the property's value, you must restart the
+ nodemanager in order for it to have an effect.
+
+ The roots of Yarn applications' work directories is configurable with
+ the yarn.nodemanager.local-dirs property (see below), and the roots
+ of the Yarn applications' log directories is configurable with the
+ yarn.nodemanager.log-dirs property (see also below).
+ </description>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>600</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>master.docker-hadoop-cluster-network:8032</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>master.docker-hadoop-cluster-network:8030</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>master.docker-hadoop-cluster-network:8031</value>
+ </property>
+
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>yarn.timeline-service.enabled</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>yarn.timeline-service.hostname</name>
+ <value>master.docker-hadoop-cluster-network</value>
+ </property>
+ <property>
+ <name>yarn.timeline-service.generic-application-history.enabled</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.webapp.ui2.enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.principal</name>
+ <value>yarn/_HOST@EXAMPLE.COM</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.keytab</name>
+ <value>/etc/security/keytabs/yarn.keytab</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.principal</name>
+ <value>yarn/_HOST@EXAMPLE.COM</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.keytab</name>
+ <value>/etc/security/keytabs/yarn.keytab</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.principal</name>
+ <value>yarn/_HOST@EXAMPLE.COM</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.keytab</name>
+ <value>/etc/security/keytabs/yarn.keytab</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.http-authentication.type</name>
+ <value>kerberos</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.http-authentication.kerberos.principal</name>
+ <value>HTTP/_HOST@EXAMPLE.COM</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.http-authentication.kerberos.keytab</name>
+ <value>/etc/security/keytabs/yarn.keytab</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.path</name>
+ <value>/usr/local/hadoop/bin/container-executor</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.group</name>
+ <value>yarn</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.local-dirs</name>
+ <value>file:///hadoop-data/nm-local-dirs</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-dirs</name>
+ <value>file:///hadoop-data/nm-log-dirs</value>
+ </property>
+</configuration>
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml
new file mode 100644
index 00000000000..1ce0651a2a1
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml
@@ -0,0 +1,74 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+version: '3.5'
+
+networks:
+ docker-hadoop-cluster-network:
+ name: docker-hadoop-cluster-network
+
+services:
+ kdc:
+ container_name: "kdc"
+ hostname: kdc.kerberos.com
+ image: sequenceiq/kerberos
+ networks:
+ - docker-hadoop-cluster-network
+ environment:
+ REALM: EXAMPLE.COM
+ DOMAIN_REALM: kdc.kerberos.com
+
+ master:
+ image: ${DOCKER_HADOOP_IMAGE_NAME:-flink/docker-hadoop-secure-cluster:latest}
+ command: master
+ depends_on:
+ - kdc
+ container_name: "master"
+ hostname: master.docker-hadoop-cluster-network
+ networks:
+ - docker-hadoop-cluster-network
+ environment:
+ KRB_REALM: EXAMPLE.COM
+ DOMAIN_REALM: kdc.kerberos.com
+
+ slave1:
+ image: ${DOCKER_HADOOP_IMAGE_NAME:-flink/docker-hadoop-secure-cluster:latest}
+ command: worker
+ depends_on:
+ - kdc
+ - master
+ container_name: "slave1"
+ hostname: slave1.docker-hadoop-cluster-network
+ networks:
+ - docker-hadoop-cluster-network
+ environment:
+ KRB_REALM: EXAMPLE.COM
+ DOMAIN_REALM: kdc.kerberos.com
+
+ slave2:
+ image: ${DOCKER_HADOOP_IMAGE_NAME:-flink/docker-hadoop-secure-cluster:latest}
+ command: worker
+ depends_on:
+ - kdc
+ - master
+ container_name: "slave2"
+ hostname: slave2.docker-hadoop-cluster-network
+ networks:
+ - docker-hadoop-cluster-network
+ environment:
+ KRB_REALM: EXAMPLE.COM
+ DOMAIN_REALM: kdc.kerberos.com
diff --git a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
new file mode 100755
index 00000000000..d6f2a880112
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+set -o pipefail
+
+source "$(dirname "$0")"/common.sh
+
+FLINK_TARBALL_DIR=$TEST_DATA_DIR
+FLINK_TARBALL=flink.tar.gz
+FLINK_DIRNAME=$(basename $FLINK_DIR)
+
+echo "Flink Tarball directory $FLINK_TARBALL_DIR"
+echo "Flink tarball filename $FLINK_TARBALL"
+echo "Flink distribution directory name $FLINK_DIRNAME"
+echo "End-to-end directory $END_TO_END_DIR"
+docker --version
+docker-compose --version
+
+mkdir -p $FLINK_TARBALL_DIR
+tar czf $FLINK_TARBALL_DIR/$FLINK_TARBALL -C $(dirname $FLINK_DIR) .
+
+echo "Building Hadoop Docker container"
+until docker build --build-arg HADOOP_VERSION=2.8.4 -f $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/Dockerfile -t flink/docker-hadoop-secure-cluster:latest $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/; do
+ # with all the downloading and ubuntu updating a lot of flakiness can happen, make sure
+ # we don't immediately fail
+ echo "Something went wrong while building the Docker image, retrying ..."
+ sleep 2
+done
+
+echo "Starting Hadoop cluster"
+docker-compose -f $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml up -d
+
+# make sure we stop our cluster at the end
+function cluster_shutdown {
+ # don't call ourselves again for another signal interruption
+ trap "exit -1" INT
+ # don't call ourselves again for normal exit
+ trap "" EXIT
+
+ docker-compose -f $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml down
+ rm $FLINK_TARBALL_DIR/$FLINK_TARBALL
+}
+trap cluster_shutdown INT
+trap cluster_shutdown EXIT
+
+until docker cp $FLINK_TARBALL_DIR/$FLINK_TARBALL master:/home/hadoop-user/; do
+ # we're retrying this one because we don't know yet if the container is ready
+ echo "Uploading Flink tarball to docker master failed, retrying ..."
+ sleep 5
+done
+
+# now, at least the container is ready
+docker exec -it master bash -c "tar xzf /home/hadoop-user/$FLINK_TARBALL --directory /home/hadoop-user/"
+
+# minimal Flink config, bebe
+docker exec -it master bash -c "echo \"security.kerberos.login.keytab: /home/hadoop-user/hadoop-user.keytab\" > /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
+docker exec -it master bash -c "echo \"security.kerberos.login.principal: hadoop-user\" >> /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
+
+echo "Flink config:"
+docker exec -it master bash -c "cat /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
+
+# make the output path random, just in case it already exists, for example if we
+# had cached docker containers
+OUTPUT_PATH=hdfs:///user/hadoop-user/wc-out-$RANDOM
+
+# it's important to run this with higher parallelism, otherwise we might risk that
+# JM and TM are on the same YARN node and that we therefore don't test the keytab shipping
+until docker exec -it master bash -c "export HADOOP_CLASSPATH=\`hadoop classpath\` && /home/hadoop-user/$FLINK_DIRNAME/bin/flink run -m yarn-cluster -yn 3 -ys 1 -ytm 1200 -yjm 800 -p 3 /home/hadoop-user/$FLINK_DIRNAME/examples/streaming/WordCount.jar --output $OUTPUT_PATH"; do
+ echo "Running the Flink job failed, might be that the cluster is not ready yet, retrying ..."
+ sleep 5
+done
+
+docker exec -it master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab hadoop-user"
+OUTPUT=$(docker exec -it master bash -c "hdfs dfs -cat $OUTPUT_PATH/*")
+docker exec -it master bash -c "kdestroy"
+echo "$OUTPUT"
+
+if [[ ! "$OUTPUT" =~ "consummation,1" ]]; then
+ echo "Output does not contain (consummation, 1) as required"
+ exit 1
+fi
+
+if [[ ! "$OUTPUT" =~ "of,14" ]]; then
+ echo "Output does not contain (of, 14) as required"
+ exit 1
+fi
+
+if [[ ! "$OUTPUT" =~ "calamity,1" ]]; then
+ echo "Output does not contain (calamity, 1) as required"
+ exit 1
+fi
+
+echo "Running Job without configured keytab, the exception you see below is expected"
+docker exec -it master bash -c "echo \"\" > /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
+# verify that it doesn't work if we don't configure a keytab
+OUTPUT=$(docker exec -it master bash -c "export HADOOP_CLASSPATH=\`hadoop classpath\` && /home/hadoop-user/$FLINK_DIRNAME/bin/flink run -m yarn-cluster -yn 3 -ys 1 -ytm 1200 -yjm 800 -p 3 /home/hadoop-user/$FLINK_DIRNAME/examples/streaming/WordCount.jar --output $OUTPUT_PATH")
+echo "$OUTPUT"
+
+if [[ ! "$OUTPUT" =~ "Hadoop security with Kerberos is enabled but the login user does not have Kerberos credentials" ]]; then
+ echo "Output does not contain the Kerberos error message as required"
+ exit 1
+fi
diff --git a/pom.xml b/pom.xml
index 3540215b29f..6982a228ccd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1115,6 +1115,7 @@ under the License.
<exclude>flink-libraries/flink-table/src/test/scala/resources/*.out</exclude>
<exclude>flink-yarn/src/test/resources/krb5.keytab</exclude>
<exclude>flink-end-to-end-tests/test-scripts/test-data/*</exclude>
+ <exclude>flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/keystore.jks</exclude>
<!-- snapshots -->
<exclude>**/src/test/resources/*-snapshot</exclude>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services