You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by al...@apache.org on 2018/07/26 11:20:38 UTC
[flink] 02/02: [FLINK-8981] Add end-to-end test for running on YARN
with Kerberos
This is an automated email from the ASF dual-hosted git repository.
aljoscha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 958e88cfdf3799b92e14b58f69081674fdbf2bdf
Author: Aljoscha Krettek <al...@gmail.com>
AuthorDate: Wed Jul 18 13:46:29 2018 +0200
[FLINK-8981] Add end-to-end test for running on YARN with Kerberos
This adds a complete Docker container setup and Docker Compose file for
starting a kerberized Hadoop cluster on Docker.
The test script does the following:
* package "build-target" Flink dist into a tarball
* build docker container
* start cluster using docker compose
* upload tarball and unpack
* modify flink-conf.yaml to use Kerberos keytab for hadoop-user
* Run Streaming WordCount Job
* verify results
We set an exit trap before to ensure that we shut down the docker
compose cluster at the end.
---
flink-end-to-end-tests/run-nightly-tests.sh | 2 +
.../docker-hadoop-secure-cluster/Dockerfile | 158 +++++++++++++++++
.../docker-hadoop-secure-cluster/README.md | 118 +++++++++++++
.../docker-hadoop-secure-cluster/bootstrap.sh | 134 +++++++++++++++
.../config/container-executor.cfg | 23 +++
.../config/core-site.xml | 68 ++++++++
.../config/hdfs-site.xml | 190 +++++++++++++++++++++
.../config/keystore.jks | Bin 0 -> 636 bytes
.../docker-hadoop-secure-cluster/config/krb5.conf | 40 +++++
.../config/mapred-site.xml | 52 ++++++
.../docker-hadoop-secure-cluster/config/ssh_config | 23 +++
.../config/ssl-client.xml | 80 +++++++++
.../config/ssl-server.xml | 77 +++++++++
.../config/yarn-site.xml | 166 ++++++++++++++++++
.../docker-compose.yml | 74 ++++++++
.../test-scripts/test_yarn_kerberos_docker.sh | 137 +++++++++++++++
pom.xml | 1 +
17 files changed, 1343 insertions(+)
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index 11f1827..960ba89 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -108,5 +108,7 @@ run_test "Avro Confluent Schema Registry nightly end-to-end test" "$END_TO_END_D
run_test "State TTL Heap backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh file"
run_test "State TTL RocksDb backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh rocks"
+run_test "Running Kerberized YARN on Docker test " "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh"
+
printf "\n[PASS] All tests passed\n"
exit 0
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/Dockerfile b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/Dockerfile
new file mode 100644
index 0000000..806134d
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/Dockerfile
@@ -0,0 +1,158 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+#
+# This image is modified version of Knappek/docker-hadoop-secure
+# * Knappek/docker-hadoop-secure <https://github.com/Knappek/docker-hadoop-secure>
+#
+# With bits and pieces added from Lewuathe/docker-hadoop-cluster to extend it to start a proper kerberized Hadoop cluster:
+# * Lewuathe/docker-hadoop-cluster <https://github.com/Lewuathe/docker-hadoop-cluster>
+#
+# Creates multi-node, kerberized Hadoop cluster on Docker
+
+FROM sequenceiq/pam:ubuntu-14.04
+
+RUN set -x \
+ && addgroup hadoop \
+ && useradd -d /home/hdfs -ms /bin/bash -G hadoop -p hdfs hdfs \
+ && useradd -d /home/yarn -ms /bin/bash -G hadoop -p yarn yarn \
+ && useradd -d /home/mapred -ms /bin/bash -G hadoop -p mapred mapred \
+ && useradd -d /home/hadoop-user -ms /bin/bash -p hadoop-user hadoop-user
+
+# install dev tools
+RUN set -x \
+ && apt-get update && apt-get install -y \
+ curl tar sudo openssh-server openssh-client rsync unzip krb5-user
+
+# Kerberos client
+RUN set -x \
+ && mkdir -p /var/log/kerberos \
+ && touch /var/log/kerberos/kadmind.log
+
+# passwordless ssh
+RUN set -x \
+ && rm -f /etc/ssh/ssh_host_dsa_key /etc/ssh/ssh_host_rsa_key /root/.ssh/id_rsa \
+ && ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key \
+ && ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key \
+ && ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa \
+ && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+
+# java
+RUN set -x \
+ && mkdir -p /usr/java/default \
+ && curl -Ls 'http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz' -H 'Cookie: oraclelicense=accept-securebackup-cookie' | \
+ tar --strip-components=1 -xz -C /usr/java/default/
+
+ENV JAVA_HOME /usr/java/default
+ENV PATH $PATH:$JAVA_HOME/bin
+
+RUN set -x \
+ && curl -LOH 'Cookie: oraclelicense=accept-securebackup-cookie' 'http://download.oracle.com/otn-pub/java/jce/8/jce_policy-8.zip' \
+ && unzip jce_policy-8.zip \
+ && cp /UnlimitedJCEPolicyJDK8/local_policy.jar /UnlimitedJCEPolicyJDK8/US_export_policy.jar $JAVA_HOME/jre/lib/security
+
+ARG HADOOP_VERSION=2.8.4
+
+ENV HADOOP_URL http://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
+RUN set -x \
+ && curl -fSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz \
+ && tar -xf /tmp/hadoop.tar.gz -C /usr/local/ \
+ && rm /tmp/hadoop.tar.gz*
+
+WORKDIR /usr/local
+RUN set -x \
+ && ln -s /usr/local/hadoop-${HADOOP_VERSION} /usr/local/hadoop \
+ && chown root:root -R /usr/local/hadoop-${HADOOP_VERSION}/ \
+ && chown root:root -R /usr/local/hadoop/ \
+ && chown root:yarn /usr/local/hadoop/bin/container-executor \
+ && chmod 6050 /usr/local/hadoop/bin/container-executor \
+ && mkdir -p /hadoop-data/nm-local-dirs \
+ && mkdir -p /hadoop-data/nm-log-dirs \
+ && chown yarn:yarn /hadoop-data \
+ && chown yarn:yarn /hadoop-data/nm-local-dirs \
+ && chown yarn:yarn /hadoop-data/nm-log-dirs \
+ && chmod 755 /hadoop-data \
+ && chmod 755 /hadoop-data/nm-local-dirs \
+ && chmod 755 /hadoop-data/nm-log-dirs
+
+ENV HADOOP_HOME /usr/local/hadoop
+ENV HADOOP_COMMON_HOME /usr/local/hadoop
+ENV HADOOP_HDFS_HOME /usr/local/hadoop
+ENV HADOOP_MAPRED_HOME /usr/local/hadoop
+ENV HADOOP_YARN_HOME /usr/local/hadoop
+ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop
+ENV YARN_CONF_DIR /usr/local/hadoop/etc/hadoop
+ENV HADOOP_LOG_DIR /var/log/hadoop
+ENV HADOOP_BIN_HOME $HADOOP_HOME/bin
+ENV PATH $PATH:$HADOOP_BIN_HOME
+
+ENV KRB_REALM EXAMPLE.COM
+ENV DOMAIN_REALM example.com
+ENV KERBEROS_ADMIN admin/admin
+ENV KERBEROS_ADMIN_PASSWORD admin
+ENV KEYTAB_DIR /etc/security/keytabs
+
+RUN mkdir /var/log/hadoop
+
+ADD config/core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml
+ADD config/hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml
+ADD config/mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml
+ADD config/yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml
+ADD config/container-executor.cfg $HADOOP_HOME/etc/hadoop/container-executor.cfg
+ADD config/krb5.conf /etc/krb5.conf
+ADD config/ssl-server.xml $HADOOP_HOME/etc/hadoop/ssl-server.xml
+ADD config/ssl-client.xml $HADOOP_HOME/etc/hadoop/ssl-client.xml
+ADD config/keystore.jks $HADOOP_HOME/lib/keystore.jks
+
+RUN set -x \
+ && chmod 400 $HADOOP_HOME/etc/hadoop/container-executor.cfg \
+ && chown root:yarn $HADOOP_HOME/etc/hadoop/container-executor.cfg
+
+ADD config/ssh_config /root/.ssh/config
+RUN set -x \
+ && chmod 600 /root/.ssh/config \
+ && chown root:root /root/.ssh/config
+
+# workingaround docker.io build error
+RUN set -x \
+ && ls -la /usr/local/hadoop/etc/hadoop/*-env.sh \
+ && chmod +x /usr/local/hadoop/etc/hadoop/*-env.sh \
+ && ls -la /usr/local/hadoop/etc/hadoop/*-env.sh
+
+# fix the 254 error code
+RUN set -x \
+ && sed -i "/^[^#]*UsePAM/ s/.*/#&/" /etc/ssh/sshd_config \
+ && echo "UsePAM no" >> /etc/ssh/sshd_config \
+ && echo "Port 2122" >> /etc/ssh/sshd_config
+
+# Hdfs ports
+EXPOSE 50470 9000 50010 50020 50070 50075 50090 50475 50091 8020
+# Mapred ports
+EXPOSE 19888
+# Yarn ports
+EXPOSE 8030 8031 8032 8033 8040 8042 8088 8188
+# Other ports
+EXPOSE 49707 2122
+
+ADD bootstrap.sh /etc/bootstrap.sh
+RUN chown root:root /etc/bootstrap.sh
+RUN chmod 700 /etc/bootstrap.sh
+
+ENV BOOTSTRAP /etc/bootstrap.sh
+
+ENTRYPOINT ["/etc/bootstrap.sh"]
+CMD ["-h"]
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/README.md b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/README.md
new file mode 100644
index 0000000..a9f9609
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/README.md
@@ -0,0 +1,118 @@
+# Apache Hadoop Docker image with Kerberos enabled
+
+This image is modified version of Knappek/docker-hadoop-secure
+ * Knappek/docker-hadoop-secure <https://github.com/Knappek/docker-hadoop-secure>
+
+With bits and pieces added from Lewuathe/docker-hadoop-cluster to extend it to start a proper kerberized Hadoop cluster:
+ * Lewuathe/docker-hadoop-cluster <https://github.com/Lewuathe/docker-hadoop-cluster>
+
+And a lot of added stuff for making this an actual, properly configured, kerberized cluster with proper user/permissions structure.
+
+Versions
+--------
+
+* JDK8
+* Hadoop 2.8.3
+
+Default Environment Variables
+-----------------------------
+
+| Name | Value | Description |
+| ---- | ---- | ---- |
+| `KRB_REALM` | `EXAMPLE.COM` | The Kerberos Realm, more information [here](https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html#) |
+| `DOMAIN_REALM` | `example.com` | The Kerberos Domain Realm, more information [here](https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html#) |
+| `KERBEROS_ADMIN` | `admin/admin` | The KDC admin user |
+| `KERBEROS_ADMIN_PASSWORD` | `admin` | The KDC admin password |
+
+You can simply define these variables in the `docker-compose.yml`.
+
+Run image
+---------
+
+Clone the [project](https://git-wip-us.apache.org/repos/asf/flink.git) and run
+
+```
+docker-compose up
+```
+
+Usage
+-----
+
+Get the container name with `docker ps` and login to the container with
+
+```
+docker exec -it <container-name> /bin/bash
+```
+
+
+To obtain a Kerberos ticket, execute
+
+```
+kinit -kt /home/hadoop-user/hadoop-user.keytab hadoop-user
+```
+
+Afterwards you can use `hdfs` CLI like
+
+```
+hdfs dfs -ls /
+```
+
+
+Known issues
+------------
+
+### Unable to obtain Kerberos password
+
+#### Error
+docker-compose up fails for the first time with the error
+
+```
+Login failure for nn/hadoop.docker.com@EXAMPLE.COM from keytab /etc/security/keytabs/nn.service.keytab: javax.security.auth.login.LoginException: Unable to obtain password from user
+```
+
+#### Solution
+
+Stop the containers with `docker-compose down` and start again with `docker-compose up -d`.
+
+
+### JDK 8
+
+Make sure you use download a JDK version that is still available. Old versions can be deprecated by Oracle and thus the download link won't be able anymore.
+
+Get the latest JDK8 Download URL with
+
+```
+curl -s https://lv.binarybabel.org/catalog-api/java/jdk8.json
+```
+
+### Java Keystore
+
+If the Keystore has been expired, then create a new `keystore.jks`:
+
+1. create private key
+
+```
+openssl genrsa -des3 -out server.key 1024
+```
+
+2. create csr
+
+```
+openssl req -new -key server.key -out server.csr`
+```
+
+3. remove passphrase in key
+```
+cp server.key server.key.org
+openssl rsa -in server.key.org -out server.key
+```
+
+3. create self-signed cert
+```
+openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt
+```
+
+4. create JKS and import certificate
+```
+keytool -import -keystore keystore.jks -alias CARoot -file server.crt`
+```
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/bootstrap.sh b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/bootstrap.sh
new file mode 100755
index 0000000..7b5e50b
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/bootstrap.sh
@@ -0,0 +1,134 @@
+#!/bin/bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+
+$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+
+MAX_RETRY_SECONDS=800
+
+# installing libraries if any - (resource urls added comma separated to the ACP system variable)
+cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd -
+
+# kerberos client
+sed -i "s/EXAMPLE.COM/${KRB_REALM}/g" /etc/krb5.conf
+sed -i "s/example.com/${DOMAIN_REALM}/g" /etc/krb5.conf
+
+# update config files
+sed -i "s/HOSTNAME/$(hostname -f)/g" $HADOOP_PREFIX/etc/hadoop/core-site.xml
+sed -i "s/EXAMPLE.COM/${KRB_REALM}/g" $HADOOP_PREFIX/etc/hadoop/core-site.xml
+sed -i "s#/etc/security/keytabs#${KEYTAB_DIR}#g" $HADOOP_PREFIX/etc/hadoop/core-site.xml
+
+sed -i "s/EXAMPLE.COM/${KRB_REALM}/g" $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
+sed -i "s/HOSTNAME/$(hostname -f)/g" $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
+sed -i "s#/etc/security/keytabs#${KEYTAB_DIR}#g" $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
+
+sed -i "s/EXAMPLE.COM/${KRB_REALM}/g" $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
+sed -i "s/HOSTNAME/$(hostname -f)/g" $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
+sed -i "s#/etc/security/keytabs#${KEYTAB_DIR}#g" $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
+
+sed -i "s/EXAMPLE.COM/${KRB_REALM}/g" $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
+sed -i "s/HOSTNAME/$(hostname -f)/g" $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
+sed -i "s#/etc/security/keytabs#${KEYTAB_DIR}#g" $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
+
+sed -i "s#/usr/local/hadoop/bin/container-executor#${NM_CONTAINER_EXECUTOR_PATH}#g" $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
+
+# create namenode kerberos principal and keytab
+# we retry the first call because it can happen that Kerberos is not ready in
+# time
+start_time=$(date +%s)
+until kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "addprinc -randkey hdfs/$(hostname -f)@${KRB_REALM}"; do
+ current_time=$(date +%s)
+ time_diff=$((current_time - start_time))
+
+ if [ $time_diff -ge $MAX_RETRY_SECONDS ]; then
+ echo "We tried creating a Kerberos principal for $time_diff seconds, max is $MAX_RETRY_SECONDS seconds, aborting"
+ exit 1
+ else
+ echo "Creating a Kerberos principal failed. We have been trying for $time_diff seconds, retrying ..."
+ sleep 5
+ fi
+done
+
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "addprinc -randkey mapred/$(hostname -f)@${KRB_REALM}"
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "addprinc -randkey yarn/$(hostname -f)@${KRB_REALM}"
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "addprinc -randkey HTTP/$(hostname -f)@${KRB_REALM}"
+
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "xst -k hdfs.keytab hdfs/$(hostname -f) HTTP/$(hostname -f)"
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "xst -k mapred.keytab mapred/$(hostname -f) HTTP/$(hostname -f)"
+kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "xst -k yarn.keytab yarn/$(hostname -f) HTTP/$(hostname -f)"
+
+mkdir -p ${KEYTAB_DIR}
+mv hdfs.keytab ${KEYTAB_DIR}
+mv mapred.keytab ${KEYTAB_DIR}
+mv yarn.keytab ${KEYTAB_DIR}
+chmod 400 ${KEYTAB_DIR}/hdfs.keytab
+chmod 400 ${KEYTAB_DIR}/mapred.keytab
+chmod 400 ${KEYTAB_DIR}/yarn.keytab
+chown hdfs:hadoop ${KEYTAB_DIR}/hdfs.keytab
+chown mapred:hadoop ${KEYTAB_DIR}/mapred.keytab
+chown yarn:hadoop ${KEYTAB_DIR}/yarn.keytab
+
+service ssh start
+
+if [ "$1" == "--help" -o "$1" == "-h" ]; then
+ echo "Usage: $(basename $0) (master|worker)"
+ exit 0
+elif [ "$1" == "master" ]; then
+ yes| sudo -E -u hdfs $HADOOP_PREFIX/bin/hdfs namenode -format
+
+ nohup sudo -E -u hdfs $HADOOP_PREFIX/bin/hdfs namenode 2>> /var/log/hadoop/namenode.err >> /var/log/hadoop/namenode.out &
+ nohup sudo -E -u yarn $HADOOP_PREFIX/bin/yarn resourcemanager 2>> /var/log/hadoop/resourcemanager.err >> /var/log/hadoop/resourcemanager.out &
+ nohup sudo -E -u yarn $HADOOP_PREFIX/bin/yarn timelineserver 2>> /var/log/hadoop/timelineserver.err >> /var/log/hadoop/timelineserver.out &
+ nohup sudo -E -u mapred $HADOOP_PREFIX/bin/mapred historyserver 2>> /var/log/hadoop/historyserver.err >> /var/log/hadoop/historyserver.out &
+
+
+ kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "addprinc -randkey root@${KRB_REALM}"
+ kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "xst -k /root/root.keytab root"
+
+ kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "addprinc -pw hadoop-user hadoop-user@${KRB_REALM}"
+ kadmin -p ${KERBEROS_ADMIN} -w ${KERBEROS_ADMIN_PASSWORD} -q "xst -k /home/hadoop-user/hadoop-user.keytab hadoop-user"
+ chown hadoop-user:hadoop-user /home/hadoop-user/hadoop-user.keytab
+
+ kinit -kt /root/root.keytab root
+
+ hdfs dfsadmin -safemode wait
+ while [ $? -ne 0 ]; do hdfs dfsadmin -safemode wait; done
+
+ hdfs dfs -chown hdfs:hadoop /
+ hdfs dfs -chmod 755 /
+ hdfs dfs -mkdir /tmp
+ hdfs dfs -chown hdfs:hadoop /tmp
+ hdfs dfs -chmod -R 1777 /tmp
+ hdfs dfs -mkdir /tmp/logs
+ hdfs dfs -chown yarn:hadoop /tmp/logs
+ hdfs dfs -chmod 1777 /tmp/logs
+
+ hdfs dfs -mkdir -p /user/hadoop-user
+ hdfs dfs -chown hadoop-user:hadoop-user /user/hadoop-user
+
+ kdestroy
+
+ while true; do sleep 1000; done
+elif [ "$1" == "worker" ]; then
+ nohup sudo -E -u hdfs $HADOOP_PREFIX/bin/hdfs datanode 2>> /var/log/hadoop/datanode.err >> /var/log/hadoop/datanode.out &
+ nohup sudo -E -u yarn $HADOOP_PREFIX/bin/yarn nodemanager 2>> /var/log/hadoop/nodemanager.err >> /var/log/hadoop/nodemanager.out &
+ while true; do sleep 1000; done
+fi
+
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/container-executor.cfg b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/container-executor.cfg
new file mode 100644
index 0000000..e9de347
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/container-executor.cfg
@@ -0,0 +1,23 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+yarn.nodemanager.linux-container-executor.group=yarn
+yarn.nodemanager.local-dirs=/hadoop-data/nm-local-dirs
+yarn.nodemanager.log-dirs=/hadoop-data/nm-log-dirs
+banned.users=hdfs,yarn,mapred,bin
+min.user.id=500
\ No newline at end of file
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/core-site.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/core-site.xml
new file mode 100644
index 0000000..366d130
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/core-site.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://master.docker-hadoop-cluster-network:9000</value>
+ </property>
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>kerberos</value>
+ <description> Set the authentication for the cluster.
+ Valid values are: simple or kerberos.</description>
+ </property>
+ <property>
+ <name>hadoop.security.authorization</name>
+ <value>true</value>
+ <description>Enable authorization for different protocols.</description>
+ </property>
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value>
+ RULE:[2:$1](.*)
+ DEFAULT
+ </value>
+ <description>The mapping from kerberos principal names
+ to local OS user names.</description>
+ </property>
+ <property>
+ <name>hadoop.ssl.require.client.cert</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hadoop.ssl.hostname.verifier</name>
+ <value>DEFAULT</value>
+ </property>
+ <property>
+ <name>hadoop.ssl.keystores.factory.class</name>
+ <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
+ </property>
+ <property>
+ <name>hadoop.ssl.server.conf</name>
+ <value>ssl-server.xml</value>
+ </property>
+ <property>
+ <name>hadoop.ssl.client.conf</name>
+ <value>ssl-client.xml</value>
+ </property>
+ <property>
+ <name>hadoop.rpc.protection</name>
+ <value>privacy</value>
+ </property>
+</configuration>
+
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/hdfs-site.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/hdfs-site.xml
new file mode 100644
index 0000000..a4e5ed8
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/hdfs-site.xml
@@ -0,0 +1,190 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<configuration>
+ <property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>dfs.namenode.rpc-address</name>
+ <value>master.docker-hadoop-cluster-network:9000</value>
+ </property>
+ <property>
+ <name>dfs.permissions</name>
+ <value>true</value>
+ <description> If "true", enable permission checking in
+ HDFS. If "false", permission checking is turned
+ off, but all other behavior is
+ unchanged. Switching from one parameter value to the other does
+ not change the mode, owner or group of files or
+ directories. </description>
+ </property>
+
+ <property>
+ <name>dfs.permissions.supergroup</name>
+ <value>root</value>
+ <description>The name of the group of super-users.</description>
+ </property>
+
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>100</value>
+ <description>Added to grow Queue size so that more client connections are allowed</description>
+ </property>
+
+ <property>
+ <name>ipc.server.max.response.size</name>
+ <value>5242880</value>
+ </property>
+
+ <property>
+ <name>dfs.block.access.token.enable</name>
+ <value>true</value>
+ <description> If "true", access tokens are used as capabilities
+ for accessing datanodes. If "false", no access tokens are checked on
+ accessing datanodes. </description>
+ </property>
+
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value>hdfs/_HOST@EXAMPLE.COM</value>
+ <description> Kerberos principal name for the NameNode </description>
+ </property>
+
+ <property>
+ <name>dfs.secondary.namenode.kerberos.principal</name>
+ <value>hdfs/_HOST@EXAMPLE.COM</value>
+ <description>Kerberos principal name for the secondary NameNode.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.secondary.http.address</name>
+ <value>master:50090</value>
+ <description>Address of secondary namenode web server</description>
+ </property>
+
+ <property>
+ <name>dfs.secondary.https.port</name>
+ <value>50490</value>
+ <description>The https port where secondary-namenode binds</description>
+ </property>
+
+ <property>
+ <name>dfs.web.authentication.kerberos.principal</name>
+ <value>HTTP/_HOST@EXAMPLE.COM</value>
+ <description> The HTTP Kerberos principal used by Hadoop-Auth in the HTTP endpoint.
+ The HTTP Kerberos principal MUST start with 'HTTP/' per Kerberos HTTP
+ SPNEGO specification.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.web.authentication.kerberos.keytab</name>
+ <value>/etc/security/keytabs/hdfs.keytab</value>
+ <description>The Kerberos keytab file with the credentials for the HTTP
+ Kerberos principal used by Hadoop-Auth in the HTTP endpoint.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.datanode.kerberos.principal</name>
+ <value>hdfs/_HOST@EXAMPLE.COM</value>
+ <description>
+ The Kerberos principal that the DataNode runs as. "_HOST" is replaced by the real
+ host name.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.namenode.keytab.file</name>
+ <value>/etc/security/keytabs/hdfs.keytab</value>
+ <description>
+ Combined keytab file containing the namenode service and host
+ principals.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.secondary.namenode.keytab.file</name>
+ <value>/etc/security/keytabs/hdfs.keytab</value>
+ <description>
+ Combined keytab file containing the namenode service and host
+ principals.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.datanode.keytab.file</name>
+ <value>/etc/security/keytabs/hdfs.keytab</value>
+ <description>
+ The filename of the keytab file for the DataNode.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.datanode.data.dir.perm</name>
+ <value>750</value>
+ <description>The permissions that should be there on
+ dfs.data.dir directories. The datanode will not come up if the
+ permissions are different on existing dfs.data.dir directories. If
+ the directories don't exist, they will be created with this
+ permission.</description>
+ </property>
+
+ <property>
+ <name>dfs.access.time.precision</name>
+ <value>0</value>
+ <description>The access time for HDFS file is precise upto this
+ value.The default value is 1 hour. Setting a value of 0
+ disables access times for HDFS.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cluster.administrators</name>
+ <value>root</value>
+ <description>ACL for who all can view the default servlets in the HDFS</description>
+ </property>
+
+ <property>
+ <name>ipc.server.read.threadpool.size</name>
+ <value>5</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>dfs.data.transfer.protection</name>
+ <value>authentication</value>
+ </property>
+ <property>
+ <name>dfs.encrypt.data.transfer</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.data.dir.perm</name>
+ <value>700</value>
+ </property>
+
+ <property>
+ <name>dfs.http.policy</name>
+ <value>HTTPS_ONLY</value>
+ </property>
+</configuration>
+
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/keystore.jks b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/keystore.jks
new file mode 100644
index 0000000..31fbd26
Binary files /dev/null and b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/keystore.jks differ
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/krb5.conf b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/krb5.conf
new file mode 100644
index 0000000..a1e38c9
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/krb5.conf
@@ -0,0 +1,40 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+[logging]
+ default = FILE:/var/log/kerberos/krb5libs.log
+ kdc = FILE:/var/log/kerberos/krb5kdc.log
+ admin_server = FILE:/var/log/kerberos/kadmind.log
+
+[libdefaults]
+ default_realm = EXAMPLE.COM
+ dns_lookup_realm = false
+ dns_lookup_kdc = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+
+[realms]
+ EXAMPLE.COM = {
+ kdc = kdc
+ admin_server = kdc
+ }
+
+[domain_realm]
+ .kdc = EXAMPLE.COM
+ kdc = EXAMPLE.COM
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/mapred-site.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/mapred-site.xml
new file mode 100644
index 0000000..fcde158
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/mapred-site.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<configuration>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.keytab</name>
+ <value>/etc/security/keytabs/mapred.keytab</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.principal</name>
+ <value>mapred/HOSTNAME@EXAMPLE.COM</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>master:19888</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.webapp.https.address</name>
+ <value>master:19889</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.webapp.spnego-keytab-file</name>
+ <value>/etc/security/keytabs/mapred.keytab</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.webapp.spnego-principal</name>
+ <value>HTTP/HOSTNAME@EXAMPLE.COM</value>
+ </property>
+</configuration>
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssh_config b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssh_config
new file mode 100644
index 0000000..486f953
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssh_config
@@ -0,0 +1,23 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+Host *
+ UserKnownHostsFile /dev/null
+ StrictHostKeyChecking no
+ LogLevel quiet
+ Port 2122
\ No newline at end of file
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssl-client.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssl-client.xml
new file mode 100644
index 0000000..b8c0085
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssl-client.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<configuration>
+
+<property>
+ <name>ssl.client.truststore.location</name>
+ <value>/usr/local/hadoop/lib/keystore.jks</value>
+ <description>Truststore to be used by clients like distcp. Must be
+ specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.password</name>
+ <value>bigdata</value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload check interval, in milliseconds.
+ Default value is 10000 (10 seconds).
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.location</name>
+ <value>/usr/local/hadoop/lib/keystore.jks</value>
+ <description>Keystore to be used by clients like distcp. Must be
+ specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.password</name>
+ <value>bigdata</value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.keypassword</name>
+ <value>bigdata</value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+</configuration>
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssl-server.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssl-server.xml
new file mode 100644
index 0000000..ce94ad7
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/ssl-server.xml
@@ -0,0 +1,77 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+<property>
+ <name>ssl.server.truststore.location</name>
+ <value>/usr/local/hadoop/lib/keystore.jks</value>
+ <description>Truststore to be used by NN and DN. Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.password</name>
+ <value>bigdata</value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload check interval, in milliseconds.
+ Default value is 10000 (10 seconds).
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.location</name>
+ <value>/usr/local/hadoop/lib/keystore.jks</value>
+ <description>Keystore to be used by NN and DN. Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.password</name>
+ <value>bigdata</value>
+ <description>Must be specified</description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.keypassword</name>
+ <value>bigdata</value>
+ <description>Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+</configuration>
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml
new file mode 100644
index 0000000..62bea95
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml
@@ -0,0 +1,166 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<configuration>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>/usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/*</value>
+ </property>
+
+ <property>
+ <description>
+ Number of seconds after an application finishes before the nodemanager's
+ DeletionService will delete the application's localized file directory
+ and log directory.
+
+ To diagnose Yarn application problems, set this property's value large
+ enough (for example, to 600 = 10 minutes) to permit examination of these
+ directories. After changing the property's value, you must restart the
+ nodemanager in order for it to have an effect.
+
+ The roots of Yarn applications' work directories is configurable with
+ the yarn.nodemanager.local-dirs property (see below), and the roots
+ of the Yarn applications' log directories is configurable with the
+ yarn.nodemanager.log-dirs property (see also below).
+ </description>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>600</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>master.docker-hadoop-cluster-network:8032</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>master.docker-hadoop-cluster-network:8030</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>master.docker-hadoop-cluster-network:8031</value>
+ </property>
+
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>yarn.timeline-service.enabled</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>yarn.timeline-service.hostname</name>
+ <value>master.docker-hadoop-cluster-network</value>
+ </property>
+ <property>
+ <name>yarn.timeline-service.generic-application-history.enabled</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.webapp.ui2.enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.principal</name>
+ <value>yarn/_HOST@EXAMPLE.COM</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.keytab</name>
+ <value>/etc/security/keytabs/yarn.keytab</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.principal</name>
+ <value>yarn/_HOST@EXAMPLE.COM</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.keytab</name>
+ <value>/etc/security/keytabs/yarn.keytab</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.principal</name>
+ <value>yarn/_HOST@EXAMPLE.COM</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.keytab</name>
+ <value>/etc/security/keytabs/yarn.keytab</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.http-authentication.type</name>
+ <value>kerberos</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.http-authentication.kerberos.principal</name>
+ <value>HTTP/_HOST@EXAMPLE.COM</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.http-authentication.kerberos.keytab</name>
+ <value>/etc/security/keytabs/yarn.keytab</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.path</name>
+ <value>/usr/local/hadoop/bin/container-executor</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.group</name>
+ <value>yarn</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.local-dirs</name>
+ <value>file:///hadoop-data/nm-local-dirs</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-dirs</name>
+ <value>file:///hadoop-data/nm-log-dirs</value>
+ </property>
+</configuration>
diff --git a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml
new file mode 100644
index 0000000..1ce0651
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml
@@ -0,0 +1,74 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+version: '3.5'
+
+networks:
+ docker-hadoop-cluster-network:
+ name: docker-hadoop-cluster-network
+
+services:
+ kdc:
+ container_name: "kdc"
+ hostname: kdc.kerberos.com
+ image: sequenceiq/kerberos
+ networks:
+ - docker-hadoop-cluster-network
+ environment:
+ REALM: EXAMPLE.COM
+ DOMAIN_REALM: kdc.kerberos.com
+
+ master:
+ image: ${DOCKER_HADOOP_IMAGE_NAME:-flink/docker-hadoop-secure-cluster:latest}
+ command: master
+ depends_on:
+ - kdc
+ container_name: "master"
+ hostname: master.docker-hadoop-cluster-network
+ networks:
+ - docker-hadoop-cluster-network
+ environment:
+ KRB_REALM: EXAMPLE.COM
+ DOMAIN_REALM: kdc.kerberos.com
+
+ slave1:
+ image: ${DOCKER_HADOOP_IMAGE_NAME:-flink/docker-hadoop-secure-cluster:latest}
+ command: worker
+ depends_on:
+ - kdc
+ - master
+ container_name: "slave1"
+ hostname: slave1.docker-hadoop-cluster-network
+ networks:
+ - docker-hadoop-cluster-network
+ environment:
+ KRB_REALM: EXAMPLE.COM
+ DOMAIN_REALM: kdc.kerberos.com
+
+ slave2:
+ image: ${DOCKER_HADOOP_IMAGE_NAME:-flink/docker-hadoop-secure-cluster:latest}
+ command: worker
+ depends_on:
+ - kdc
+ - master
+ container_name: "slave2"
+ hostname: slave2.docker-hadoop-cluster-network
+ networks:
+ - docker-hadoop-cluster-network
+ environment:
+ KRB_REALM: EXAMPLE.COM
+ DOMAIN_REALM: kdc.kerberos.com
diff --git a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
new file mode 100755
index 0000000..e7726c9
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+set -o pipefail
+
+source "$(dirname "$0")"/common.sh
+
+FLINK_TARBALL_DIR=$TEST_DATA_DIR
+FLINK_TARBALL=flink.tar.gz
+FLINK_DIRNAME=$(basename $FLINK_DIR)
+
+MAX_RETRY_SECONDS=800
+
+echo "Flink Tarball directory $FLINK_TARBALL_DIR"
+echo "Flink tarball filename $FLINK_TARBALL"
+echo "Flink distribution directory name $FLINK_DIRNAME"
+echo "End-to-end directory $END_TO_END_DIR"
+docker --version
+docker-compose --version
+
+mkdir -p $FLINK_TARBALL_DIR
+tar czf $FLINK_TARBALL_DIR/$FLINK_TARBALL -C $(dirname $FLINK_DIR) .
+
+echo "Building Hadoop Docker container"
+until docker build --build-arg HADOOP_VERSION=2.8.4 -f $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/Dockerfile -t flink/docker-hadoop-secure-cluster:latest $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/; do
+ # with all the downloading and ubuntu updating a lot of flakiness can happen, make sure
+ # we don't immediately fail
+ echo "Something went wrong while building the Docker image, retrying ..."
+ sleep 2
+done
+
+echo "Starting Hadoop cluster"
+docker-compose -f $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml up -d
+
+# make sure we stop our cluster at the end
+function cluster_shutdown {
+ # don't call ourselves again for another signal interruption
+ trap "exit -1" INT
+ # don't call ourselves again for normal exit
+ trap "" EXIT
+
+ docker-compose -f $END_TO_END_DIR/test-scripts/docker-hadoop-secure-cluster/docker-compose.yml down
+ rm $FLINK_TARBALL_DIR/$FLINK_TARBALL
+}
+trap cluster_shutdown INT
+trap cluster_shutdown EXIT
+
+until docker cp $FLINK_TARBALL_DIR/$FLINK_TARBALL master:/home/hadoop-user/; do
+ # we're retrying this one because we don't know yet if the container is ready
+ echo "Uploading Flink tarball to docker master failed, retrying ..."
+ sleep 5
+done
+
+# now, at least the container is ready
+docker exec -it master bash -c "tar xzf /home/hadoop-user/$FLINK_TARBALL --directory /home/hadoop-user/"
+
+# minimal Flink config, bebe
+docker exec -it master bash -c "echo \"security.kerberos.login.keytab: /home/hadoop-user/hadoop-user.keytab\" > /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
+docker exec -it master bash -c "echo \"security.kerberos.login.principal: hadoop-user\" >> /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
+
+echo "Flink config:"
+docker exec -it master bash -c "cat /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
+
+# make the output path random, just in case it already exists, for example if we
+# had cached docker containers
+OUTPUT_PATH=hdfs:///user/hadoop-user/wc-out-$RANDOM
+
+start_time=$(date +%s)
+# it's important to run this with higher parallelism, otherwise we might risk that
+# JM and TM are on the same YARN node and that we therefore don't test the keytab shipping
+until docker exec -it master bash -c "export HADOOP_CLASSPATH=\`hadoop classpath\` && /home/hadoop-user/$FLINK_DIRNAME/bin/flink run -m yarn-cluster -yn 3 -ys 1 -ytm 1200 -yjm 800 -p 3 /home/hadoop-user/$FLINK_DIRNAME/examples/streaming/WordCount.jar --output $OUTPUT_PATH"; do
+ current_time=$(date +%s)
+ time_diff=$((current_time - start_time))
+
+ if [ $time_diff -ge $MAX_RETRY_SECONDS ]; then
+ echo "We tried running the job for $time_diff seconds, max is $MAX_RETRY_SECONDS seconds, aborting"
+ mkdir -p $TEST_DATA_DIR/logs
+ echo "Hadoop logs:"
+ docker cp master:/var/log/hadoop/* $TEST_DATA_DIR/logs/
+ for f in $TEST_DATA_DIR/logs/*; do
+ echo "$f:"
+ cat $f
+ done
+ echo "Docker logs:"
+ docker logs master
+ exit 1
+ else
+ echo "Running the Flink job failed, might be that the cluster is not ready yet. We have been trying for $time_diff seconds, retrying ..."
+ sleep 5
+ fi
+done
+
+docker exec -it master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab hadoop-user"
+OUTPUT=$(docker exec -it master bash -c "hdfs dfs -cat $OUTPUT_PATH/*")
+docker exec -it master bash -c "kdestroy"
+echo "$OUTPUT"
+
+if [[ ! "$OUTPUT" =~ "consummation,1" ]]; then
+ echo "Output does not contain (consummation, 1) as required"
+ exit 1
+fi
+
+if [[ ! "$OUTPUT" =~ "of,14" ]]; then
+ echo "Output does not contain (of, 14) as required"
+ exit 1
+fi
+
+if [[ ! "$OUTPUT" =~ "calamity,1" ]]; then
+ echo "Output does not contain (calamity, 1) as required"
+ exit 1
+fi
+
+echo "Running Job without configured keytab, the exception you see below is expected"
+docker exec -it master bash -c "echo \"\" > /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
+# verify that it doesn't work if we don't configure a keytab
+OUTPUT=$(docker exec -it master bash -c "export HADOOP_CLASSPATH=\`hadoop classpath\` && /home/hadoop-user/$FLINK_DIRNAME/bin/flink run -m yarn-cluster -yn 3 -ys 1 -ytm 1200 -yjm 800 -p 3 /home/hadoop-user/$FLINK_DIRNAME/examples/streaming/WordCount.jar --output $OUTPUT_PATH")
+echo "$OUTPUT"
+
+if [[ ! "$OUTPUT" =~ "Hadoop security with Kerberos is enabled but the login user does not have Kerberos credentials" ]]; then
+ echo "Output does not contain the Kerberos error message as required"
+ exit 1
+fi
diff --git a/pom.xml b/pom.xml
index f56d557..97d8ad4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1115,6 +1115,7 @@ under the License.
<exclude>flink-libraries/flink-table/src/test/scala/resources/*.out</exclude>
<exclude>flink-yarn/src/test/resources/krb5.keytab</exclude>
<exclude>flink-end-to-end-tests/test-scripts/test-data/*</exclude>
+ <exclude>flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/keystore.jks</exclude>
<!-- snapshots -->
<exclude>**/src/test/resources/*-snapshot</exclude>