You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2019/04/19 17:31:11 UTC
[hadoop] branch trunk updated: YARN-9254. Add support for storing
application catalog data to HDFS. Contributed by Eric Yang
This is an automated email from the ASF dual-hosted git repository.
billie pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 317fcba YARN-9254. Add support for storing application catalog data to HDFS. Contributed by Eric Yang
317fcba is described below
commit 317fcbabe30cee90216e71e4615777459ef3a497
Author: Billie Rinaldi <bi...@apache.org>
AuthorDate: Fri Apr 19 10:29:57 2019 -0700
YARN-9254. Add support for storing application catalog data to HDFS. Contributed by Eric Yang
---
.../src/main/scripts/entrypoint.sh | 50 ++++++++++++++------
.../src/site/markdown/yarn-service/Examples.md | 54 ++++++++++++++++++++++
2 files changed, 91 insertions(+), 13 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/src/main/scripts/entrypoint.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/src/main/scripts/entrypoint.sh
index dfbd7e3..e8ab488 100755
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/src/main/scripts/entrypoint.sh
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/src/main/scripts/entrypoint.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -x
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -32,25 +32,49 @@ template_generator() {
export JAVA_HOME=/usr/lib/jvm/jre
export HADOOP_CONF_DIR=/etc/hadoop/conf
-/opt/apache/solr/bin/solr start -p 8983 -force
-/opt/apache/solr/bin/solr create_core -c appcatalog -force
-/opt/apache/solr/bin/post -c appcatalog /tmp/samples.xml
-if [ -d /etc/hadoop/conf ]; then
- sed -i.bak 's/shared.loader=.*$/shared.loader=\/etc\/hadoop\/conf/g' /etc/tomcat/catalina.properties
+
+SOLR_OPTS=()
+
+if [ "${SOLR_STORAGE_TYPE}" == "hdfs" ]; then
+ SOLR_OPTS+=("-Dsolr.directoryFactory=HdfsDirectoryFactory")
+ SOLR_OPTS+=("-Dsolr.lock.type=hdfs")
+ if [ -e "$HADOOP_CONF_DIR" ]; then
+ SOLR_OPTS+=("-Dsolr.hdfs.confdir=${HADOOP_CONF_DIR}")
+ fi
fi
+
+if [ "${SOLR_DATA_DIR}" != "" ]; then
+ SOLR_OPTS+=("-Dsolr.data.dir=$SOLR_DATA_DIR")
+ fi
+
if [ -e "$KEYTAB" ]; then
+ SOLR_OPTS+=("-Dsolr.hdfs.security.kerberos.enabled=true")
+ SOLR_OPTS+=("-Dsolr.hdfs.security.kerberos.keytabfile=${KEYTAB}")
+ SOLR_OPTS+=("-Dsolr.hdfs.security.kerberos.principal=${PRINCIPAL}")
export JAVA_OPTS="$JAVA_OPTS -Djava.security.auth.login.config=/etc/tomcat/jaas.config -Djava.security.krb5.conf=/etc/krb5.conf -Djavax.security.auth.useSubjectCredsOnly=false"
template_generator /etc/tomcat/jaas.config.template /etc/tomcat/jaas.config
fi
+
+export SOLR_OPTS
+
+/opt/apache/solr/bin/solr start "${SOLR_OPTS[@]}" -p 8983 -force
+/opt/apache/solr/bin/solr create_core -c appcatalog -force
+/opt/apache/solr/bin/post -c appcatalog /tmp/samples.xml
+if [ -d /etc/hadoop/conf ]; then
+ sed -i.bak 's/shared.loader=.*$/shared.loader=\/etc\/hadoop\/conf/g' /etc/tomcat/catalina.properties
+fi
+
if [ -e "$SPNEGO_KEYTAB" ]; then
sed -i.bak 's/authentication.type=.*$/authentication.type=kerberos/g' /etc/tomcat/catalina.properties
sed -i.bak 's/simple.anonymous.allowed=.*$/simple.anonymous.allowed=false/g' /etc/tomcat/catalina.properties
- if [ -z "$SPNEGO_PRINCIPAL" ]; then
- echo "kerberos.principal=HTTP/$HOSTNAME" >> /etc/tomcat/catalina.properties
- else
- echo "kerberos.principal=$SPNEGO_PRINCIPAL" >> /etc/tomcat/catalina.properties
- fi
- echo "kerberos.keytab=$SPNEGO_KEYTAB" >> /etc/tomcat/catalina.properties
- echo "hostname=$HOSTNAME" >> /etc/tomcat/catalina.properties
+ {
+ if [ -z "$SPNEGO_PRINCIPAL" ]; then
+ echo "kerberos.principal=HTTP/$HOSTNAME"
+ else
+ echo "kerberos.principal=$SPNEGO_PRINCIPAL"
+ fi
+ echo "kerberos.keytab=$SPNEGO_KEYTAB"
+ echo "hostname=$HOSTNAME"
+ } >> /etc/tomcat/catalina.properties
fi
/usr/libexec/tomcat/server start
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
index 7aae830..bf9d1f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
@@ -188,6 +188,60 @@ For secure cluster, Kerberos settings for application catalog can be configured
| KEYTAB | /etc/security/keytabs/yarn.service.ketab | Path to keytab file, used by YARN service application master. |
| PRINCIPAL | yarn/_HOST@EXAMPLE.COM | Service principal used by YARN service application master. |
+Application Catalog environment options:
+
+| Environment variables | Description |
+| KEYTAB | Service user keytab file for accessing HDFS. |
+| PRINCIPAL | Service user Kerboers principal. |
+| SOLR_DATA_DIR | Location to store Solr data. |
+| SOLR_STORAGE_TYPE | Storage type for Solr data, supported type are: hdfs, local |
+| SPNEGO_KEYTAB | Location of the keytab file used for authenticating HTTP endpoint. |
+| SPNEGO_PRINCIPAL | The Kerberos principal to be used for HTTP endpoint. The principal MUST start with 'HTTP'/ as per Kerberos HTTP SPNEGO specification. |
+
+Secure application catalog Yarnfile example:
+```
+{
+ "name": "catalog",
+ "kerberos_principal" : {
+ "principal_name" : "catalog/_HOST@EXAMPLE.COM",
+ "keytab" : "file:///etc/security/keytabs/catalog.service.keytab"
+ },
+ "version": "1",
+ "components" :
+ [
+ {
+ "name": "appcatalog",
+ "number_of_containers": 1,
+ "artifact": {
+ "id": "apache/hadoop-yarn-applications-catalog-docker:3.3.0-SNAPSHOT",
+ "type": "DOCKER"
+ },
+ "resource": {
+ "cpus": 1,
+ "memory": "256"
+ },
+ "configuration": {
+ "env": {
+ "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE":"true",
+ "YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS":"/etc/hadoop/conf:/etc/hadoop/conf:ro,/etc/krb5.conf:/etc/krb5.conf:ro,/etc/security/keytabs/catalog.service.keytab:/etc/security/keytabs/hbase.service.keytab:ro,/etc/security/keytabs/spnego.service.keytab:/etc/security/keytabs/spnego.service.keytab:ro",
+ "SPNEGO_KEYTAB":"/etc/security/keytabs/spnego.service.keytab",
+ "SPNEGO_PRINCIPAL":"HTTP/host-3.example.com@EXAMPLE.COM",
+ "KEYTAB":"/etc/security/keytabs/catalog.service.keytab",
+ "PRINCIPAL":"catalog/host3.example.com@EXAMPLE.COM",
+ "SOLR_DATA_DIR":"hdfs://host-1.example.com:9000/tmp/solr",
+ "SOLR_UPDATE_LOG":"hdfs://host-1.example.com:9000/tmp/solr",
+ "SOLR_STORAGE_TYPE":"hdfs"
+ },
+ "properties": {
+ "docker.network": "host"
+ }
+ }
+ }
+ ]
+}
+
+```
+
## Docker image ENTRYPOINT support
Docker images may have built with ENTRYPOINT to enable start up of docker image without any parameters.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org