You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ji...@apache.org on 2024/09/07 09:45:05 UTC
(ambari) 14/22: add kyuubi
This is an automated email from the ASF dual-hosted git repository.
jialiang pushed a commit to branch eee
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 830b4d47f3609f3c11af8085e2936edcf9f77111
Author: jialiang <ji...@gmail.com>
AuthorDate: Sat Sep 7 10:59:35 2024 +0800
add kyuubi
---
.../BIGTOP/3.2.0/services/KYUUBI/alerts.json | 29 +++
.../KYUUBI/configuration/kyuubi-defaults.xml | 136 ++++++++++++++
.../services/KYUUBI/configuration/kyuubi-env.xml | 101 ++++++++++
.../configuration/kyuubi-log4j2-properties.xml | 86 +++++++++
.../BIGTOP/3.2.0/services/KYUUBI/kerberos.json | 76 ++++++++
.../BIGTOP/3.2.0/services/KYUUBI/metainfo.xml | 162 ++++++++++++++++
.../scripts/alerts/alert_kyuubi_server_port.py | 163 ++++++++++++++++
.../KYUUBI/package/scripts/kyuubi_client.py | 70 +++++++
.../KYUUBI/package/scripts/kyuubi_server.py | 115 ++++++++++++
.../services/KYUUBI/package/scripts/params.py | 181 ++++++++++++++++++
.../KYUUBI/package/scripts/service_check.py | 67 +++++++
.../services/KYUUBI/quicklinks/quicklinks.json | 21 +++
.../3.2.0/services/KYUUBI/role_command_order.json | 7 +
.../3.2.0/services/KYUUBI/service_advisor.py | 206 +++++++++++++++++++++
14 files changed, 1420 insertions(+)
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/alerts.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/alerts.json
new file mode 100755
index 0000000000..d09e83a490
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/alerts.json
@@ -0,0 +1,29 @@
+{
+ "KYUUBI": {
+ "service": [],
+ "KYUUBI_SEVER": [
+ {
+ "name": "kyuubi_server_status",
+ "label": "Kyuubi Server",
+ "description": "This host-level alert is triggered if the Kyuubi Server cannot be determined to be up.",
+ "interval": 1,
+ "scope": "ANY",
+ "source": {
+ "type": "SCRIPT",
+ "path": "BIGTOP/3.2.0/services/KYUUBI/package/scripts/alerts/alert_kyuubi_server_port.py",
+ "parameters": [
+ {
+ "name": "check.command.timeout",
+ "display_name": "Command Timeout",
+ "value": 120.0,
+ "type": "NUMERIC",
+ "description": "The maximum time before check command will be killed by timeout",
+ "units": "seconds",
+ "threshold": "CRITICAL"
+ }
+ ]
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/configuration/kyuubi-defaults.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/configuration/kyuubi-defaults.xml
new file mode 100755
index 0000000000..2e9d776dac
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/configuration/kyuubi-defaults.xml
@@ -0,0 +1,136 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+<!-- 会起端口冲突,不作为默认配置项-->
+<!-- <property>-->
+<!-- <name>kyuubi.frontend.rest.bind.port</name>-->
+<!-- <value>10099</value>-->
+<!-- <description>-->
+<!-- Port of the machine on which to run the REST frontend service.-->
+<!-- </description>-->
+<!-- <on-ambari-upgrade add="true"/>-->
+<!-- </property>-->
+<!-- <property>-->
+<!-- <name>kyuubi.frontend.thrift.binary.bind.port</name>-->
+<!-- <value>10009</value>-->
+<!-- <description>-->
+<!-- Port of the machine on which to run the thrift frontend service via binary protocol.-->
+<!-- </description>-->
+<!-- <on-ambari-upgrade add="true"/>-->
+<!-- </property>-->
+ <property>
+ <name>kyuubi.ha.zookeeper.session.timeout</name>
+ <value>600000</value>
+ <description>
+ The timeout(ms) of a connected session to be idled
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>kyuubi.session.engine.initialize.timeout</name>
+ <value>300000</value>
+ <description>
+ Timeout for starting the background engine, e.g. SparkSQLEngine.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>kyuubi.authentication</name>
+ <value>{{kyuubi_authentication}}</value>
+ <description>
+ Client authentication types.
+NOSASL: raw transport.
+NONE: no authentication check.
+KERBEROS: Kerberos/GSSAPI authentication.
+LDAP: Lightweight Directory Access Protocol authentication.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spark.master</name>
+ <value>yarn</value>
+ <description>
+ The deploying mode of spark application.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spark.submit.deployMode</name>
+ <value>client</value>
+ <description>spark submit deploy mode</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>spark.yarn.queue</name>
+ <value>default</value>
+ <description>
+ The name of the YARN queue to which the application is submitted.
+ </description>
+ <depends-on>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>spark.yarn.driver.memory</name>
+ <value>1g</value>
+ <description>spark yarn driver momory</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>spark.executor.memory</name>
+ <value>1g</value>
+ <description>spark.executor.memory</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+
+ <property>
+ <name>kyuubi.ha.client.class</name>
+ <value>org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient</value>
+ <description>Class name for service discovery client.
+Zookeeper: org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient
+Etcd: org.apache.kyuubi.ha.client.etcd.EtcdDiscoveryClient</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>kyuubi.ha.addresses</name>
+ <value>{{cluster_zookeeper_quorum}}</value>
+ <description>The connection string for the discovery ensemble</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>kyuubi.ha.zookeeper.quorum</name>
+ <value>{{cluster_zookeeper_quorum}}</value>
+ <description>(deprecated) The connection string for the zookeeper ensemble</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>kyuubi.ha.namespace</name>
+ <value>kyuubi</value>
+ <description>The root directory for the service to deploy its instance uri</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+</configuration>
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/configuration/kyuubi-env.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/configuration/kyuubi-env.xml
new file mode 100755
index 0000000000..c8ddca945b
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/configuration/kyuubi-env.xml
@@ -0,0 +1,101 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <property>
+ <name>kyuubi_user</name>
+ <display-name>Kyuubi User</display-name>
+ <value>hive</value>
+ <property-type>USER</property-type>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ <user-groups>
+ <property>
+ <type>cluster-env</type>
+ <name>user_group</name>
+ </property>
+ <property>
+ <type>kyuubi-env</type>
+ <name>kyuubi_group</name>
+ </property>
+ </user-groups>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>kyuubi_group</name>
+ <display-name>Kyuubi Group</display-name>
+ <value>hadoop</value>
+ <property-type>GROUP</property-type>
+ <description>kyuubi group</description>
+ <value-attributes>
+ <type>user</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>kyuubi_log_dir</name>
+ <display-name>Kyuubi Log directory</display-name>
+ <value>/var/log/kyuubi</value>
+ <description>Kyuubi Log Dir</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>kyuubi_pid_dir</name>
+ <display-name>Kyuubi PID directory</display-name>
+ <value>/var/run/kyuubi</value>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>kyuubi_spark_exetension_enabled</name>
+ <display-name>Enable Extensions for Spark</display-name>
+ <value>true</value>
+ <description> Z-Ordering Support,Auxiliary Optimization Rules,Auxiliary SQL Functions.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <!-- kyuubi-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for kyuubi-env.sh file</description>
+ <value>
+#!/usr/bin/env bash
+
+export JAVA_HOME={{java_home}}
+export HADOOP_CONF_DIR=/etc/hadoop/conf
+export SPARK_HOME=/usr/bigtop/current/spark-client
+export SPARK_CONF_DIR=/etc/spark/conf
+export KYUUBI_LOG_DIR={{kyuubi_log_dir}}
+export KYUUBI_PID_DIR={{kyuubi_pid_dir}}
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/configuration/kyuubi-log4j2-properties.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/configuration/kyuubi-log4j2-properties.xml
new file mode 100755
index 0000000000..f20a607778
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/configuration/kyuubi-log4j2-properties.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <!-- kyuubi-env.sh -->
+ <property>
+ <name>content</name>
+ <description>kyuubi-log4j2-properties</description>
+ <value>
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file
+rootLogger.level = info
+rootLogger.appenderRef.stdout.ref = STDOUT
+
+# Console Appender
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.target = SYSTEM_OUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
+
+appender.console.filter.1.type = Filters
+
+appender.console.filter.1.a.type = ThresholdFilter
+appender.console.filter.1.a.level = info
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+appender.console.filter.1.b.type = RegexFilter
+appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.*
+appender.console.filter.1.b.onMatch = deny
+appender.console.filter.1.b.onMismatch = neutral
+
+# Set the default kyuubi-ctl log level to WARN. When running the kyuubi-ctl, the
+# log level for this class is used to overwrite the root logger's log level.
+logger.ctl.name = org.apache.kyuubi.ctl.ServiceControlCli
+logger.ctl.level = error
+
+# Analysis MySQLFrontend protocol traffic
+# logger.mysql.name = org.apache.kyuubi.server.mysql.codec
+# logger.mysql.level = trace
+
+# Kyuubi BeeLine
+logger.beeline.name = org.apache.hive.beeline.KyuubiBeeLine
+logger.beeline.level = error
+
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/kerberos.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/kerberos.json
new file mode 100755
index 0000000000..9235c59cf7
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/kerberos.json
@@ -0,0 +1,76 @@
+{
+ "services": [
+ {
+ "name": "KYUUBI",
+ "identities": [
+ {
+ "name": "kyuubi_smokeuser",
+ "reference": "/smokeuser"
+ },
+ {
+ "name": "kyuubiuser",
+ "principal": {
+ "value": "${kyuubi-env/kyuubi_user}${principal_suffix}@${realm}",
+ "type" : "user",
+ "local_username" : "${kyuubi-env/kyuubi_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/kyuubi.headless.keytab",
+ "owner": {
+ "name": "${kyuubi-env/kyuubi_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ }
+ }
+ },
+ {
+ "name": "kyuubi_service_keytab",
+ "principal": {
+ "value": "kyuubi/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "kyuubi-defaults/kyuubi.kinit.principal",
+ "local_username" : "${kyuubi-env/kyuubi_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/kyuubi.service.keytab",
+ "owner": {
+ "name": "${kyuubi-env/kyuubi_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "kyuubi-defaults/kyuubi.kinit.keytab"
+ }
+ }
+ ],
+ "configurations": [
+ {
+ "kyuubi-defaults": {
+ "kyuubi.authentication": "KERBEROS",
+ "kyuubi.kinit.keytab": "${keytab_dir}/kyuubi.service.keytab",
+ "kyuubi.kinit.principal": "kyuubi/_HOST@${realm}",
+ "kyuubi.ha.zookeeper.acl.enabled": "true",
+ "kyuubi.ha.zookeeper.auth.keytab": "${keytab_dir}/kyuubi.service.keytab",
+ "kyuubi.ha.zookeeper.auth.principal": "kyuubi/_HOST@${realm}",
+ "kyuubi.ha.zookeeper.auth.type": "KERBEROS",
+ "kyuubi.ha.zookeeper.engine.auth.type": "KERBEROS"
+
+ }
+ }
+ ],
+ "components": [
+ {
+ "name": "KYUUBI_SERVER"
+ },
+ {
+ "name": "KYUUBI_CLIENT"
+ }
+ ]
+ }
+ ]
+}
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/metainfo.xml
new file mode 100755
index 0000000000..a92f390cee
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/metainfo.xml
@@ -0,0 +1,162 @@
+<?xml version="1.0"?>
+<!--Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>KYUUBI</name>
+ <displayName>Kyuubi</displayName>
+ <comment>AKyuubi is a distributed multi-tenant Thrift JDBC/ODBC server for large-scale data management, processing, and analytics, built on top of Apache Spark and designed to support more engines (i.e., Flink). </comment>
+ <version>1.8.1</version>
+ <components>
+ <component>
+ <name>KYUUBI_SERVER</name>
+ <displayName>Kyuubi Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ <co-locate>HIVE/HIVE_SERVER</co-locate>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>SPARK/SPARK_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/kyuubi_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ <configFiles>
+ <configFile>
+ <type>env</type>
+ <fileName>kyuubi-defaults.xml</fileName>
+ <dictionaryName>kyuubi-defaults</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>kyuubi-env.xml</fileName>
+ <dictionaryName>kyuubi-env</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>kyuubi-log4j2-properties.xml</fileName>
+ <dictionaryName>kyuubi-log4j2-properties</dictionaryName>
+ </configFile>
+ </configFiles>
+ <logs>
+ <log>
+ <logId>kyuubi_server</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
+ </component>
+
+<!-- <component>-->
+<!-- <name>KYUUBI_CLIENT</name>-->
+<!-- <displayName>Kyuubi Client</displayName>-->
+<!-- <category>CLIENT</category>-->
+<!-- <cardinality>1+</cardinality>-->
+<!-- <versionAdvertised>true</versionAdvertised>-->
+<!-- <commandScript>-->
+<!-- <script>scripts/kyuubi_client.py</script>-->
+<!-- <scriptType>PYTHON</scriptType>-->
+<!-- </commandScript>-->
+<!-- <configFiles>-->
+<!-- <configFile>-->
+<!-- <type>env</type>-->
+<!-- <fileName>kyuubi-defaults.conf</fileName>-->
+<!-- <dictionaryName>kyuubi-defaults</dictionaryName>-->
+<!-- </configFile>-->
+<!-- <configFile>-->
+<!-- <type>env</type>-->
+<!-- <fileName>kyuubi-env.sh</fileName>-->
+<!-- <dictionaryName>kyuubi-env</dictionaryName>-->
+<!-- </configFile>-->
+<!-- </configFiles>-->
+<!-- </component>-->
+
+ </components>
+
+ <configuration-dependencies>
+ <config-type>kyuubi-defaults</config-type>
+ <config-type>kyuubi-env</config-type>
+ <config-type>kyuubi-log4j2-properties</config-type>
+ </configuration-dependencies>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+
+ <requiredServices>
+ <service>ZOOKEEPER</service>
+ <service>HDFS</service>
+ <service>YARN</service>
+ <service>HIVE</service>
+ <service>SPARK</service>
+ </requiredServices>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>redhat9,redhat8,redhat7,amazonlinux2,redhat6,suse11,suse12</osFamily>
+ <packages>
+ <package>
+ <name>kyuubi_${stack_version}</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>debian7,debian9,ubuntu12,ubuntu14,ubuntu16,ubuntu18</osFamily>
+ <packages>
+ <package>
+ <name>kyuubi_${stack_version}</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ </service>
+ </services>
+</metainfo>
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/alerts/alert_kyuubi_server_port.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/alerts/alert_kyuubi_server_port.py
new file mode 100755
index 0000000000..5e9aef04dc
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/alerts/alert_kyuubi_server_port.py
@@ -0,0 +1,163 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+//TODO: 有待实现
+alert_kyuubi_server_port.py的实现逻辑可以参考alert_spark_thrift_port.py,在此不列出具体实现逻辑,原理就是定时执行beeline连接操作判断是否可以连接成功。
+
+
+"""
+
+import os
+import socket
+import time
+import logging
+import traceback
+from resource_management.libraries.functions import format
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.core.resources import Execute
+from resource_management.core import global_lock
+
+
+stack_root = Script.get_stack_root()
+
+OK_MESSAGE = "TCP OK - {0:.3f}s response on port {1}"
+CRITICAL_MESSAGE = "Connection failed on host {0}:{1} ({2})"
+
+KYUUBI_SERVER_THRIFT_PORT_KEY = '{{kyuubi-defaults/kyuubi.frontend.thrift.binary.bind.port}'
+
+SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
+
+KYUUBI_SERVER_KERBEROS_KEYTAB = '{{kyuubi-defaults/kyuubi.kinit.keytab}}'
+KYUUBI_SERVER_PRINCIPAL_KEY = '{{kyuubi-defaults/kyuubi.kinit.principal}}'
+
+# The configured Kerberos executable search paths, if any
+KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY = '{{kerberos-env/executable_search_paths}}'
+
+THRIFT_PORT_DEFAULT = 10009
+KYUUBI_SERVER_TRANSPORT_MODE_DEFAULT = 'binary'
+
+KYUUBI_USER_KEY = '{{kyuubi-env/kyuubi_user}}'
+
+CHECK_COMMAND_TIMEOUT_KEY = 'check.command.timeout'
+CHECK_COMMAND_TIMEOUT_DEFAULT = 60.0
+
+logger = logging.getLogger('ambari_alerts')
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def get_tokens():
+ """
+ Returns a tuple of tokens in the format {{site/property}} that will be used
+ to build the dictionary passed into execute
+ """
+ return (KYUUBI_SERVER_THRIFT_PORT_KEY, KYUUBI_SERVER_TRANSPORT_MODE_DEFAULT,
+ SECURITY_ENABLED_KEY, KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY, KYUUBI_USER_KEY,
+ KYUUBI_SERVER_KERBEROS_KEYTAB, KYUUBI_SERVER_PRINCIPAL_KEY )
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def execute(configurations={}, parameters={}, host_name=None):
+ """
+ Returns a tuple containing the result code and a pre-formatted result label
+
+ Keyword arguments:
+ configurations (dictionary): a mapping of configuration key to value
+ parameters (dictionary): a mapping of script parameter key to value
+ host_name (string): the name of this host where the alert is running
+ """
+
+ kyuubi_home = os.path.join(stack_root, "current", 'kyuubi-client')
+
+ if configurations is None:
+ return ('UNKNOWN', ['There were no configurations supplied to the script.'])
+
+ transport_mode = KYUUBI_SERVER_TRANSPORT_MODE_DEFAULT
+ if KYUUBI_SERVER_TRANSPORT_MODE_DEFAULT in configurations:
+ transport_mode = configurations[KYUUBI_SERVER_TRANSPORT_MODE_DEFAULT]
+
+
+ port = THRIFT_PORT_DEFAULT
+ if transport_mode.lower() == 'binary' and KYUUBI_SERVER_THRIFT_PORT_KEY in configurations:
+ port = int(configurations[KYUUBI_SERVER_THRIFT_PORT_KEY])
+
+
+ security_enabled = False
+ if SECURITY_ENABLED_KEY in configurations:
+ security_enabled = str(configurations[SECURITY_ENABLED_KEY]).upper() == 'TRUE'
+
+ kyuubi_kerberos_keytab = None
+ if KYUUBI_SERVER_KERBEROS_KEYTAB in configurations:
+ kyuubi_kerberos_keytab = configurations[KYUUBI_SERVER_KERBEROS_KEYTAB]
+
+ if host_name is None:
+ host_name = socket.getfqdn()
+
+ kyuubi_principal = None
+ if KYUUBI_SERVER_PRINCIPAL_KEY in configurations:
+ kyuubi_principal = configurations[KYUUBI_SERVER_PRINCIPAL_KEY]
+ kyuubi_principal = kyuubi_principal.replace('_HOST',host_name.lower())
+
+ # Get the configured Kerberos executable search paths, if any
+ if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
+ kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
+ else:
+ kerberos_executable_search_paths = None
+
+ kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
+
+ kyuubiuser = configurations[KYUUBI_USER_KEY]
+
+ if security_enabled:
+ kinitcmd = format("{kinit_path_local} -kt {kyuubi_kerberos_keytab} {kyuubi_principal}; ")
+ # prevent concurrent kinit
+ kinit_lock = global_lock.get_lock(global_lock.LOCK_TYPE_KERBEROS)
+ kinit_lock.acquire()
+ try:
+ Execute(kinitcmd, user=kyuubiuser)
+ finally:
+ kinit_lock.release()
+
+ result_code = None
+ try:
+ if host_name is None:
+ host_name = socket.getfqdn()
+
+ beeline_url = ["jdbc:hive2://{host_name}:{port}/default", "transportMode={transport_mode}"]
+ if security_enabled:
+ beeline_url.append("principal={kyuubi_principal}")
+
+ # append url according to used transport
+
+ beeline_cmd = os.path.join(kyuubi_home, "bin", "beeline")
+ cmd = "! %s -u '%s' -e '' 2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL' -e 'Error: Could not open'" % \
+ (beeline_cmd, format(";".join(beeline_url)))
+
+ start_time = time.time()
+ try:
+ Execute(cmd, user=kyuubiuser, path=[beeline_cmd], timeout=CHECK_COMMAND_TIMEOUT_DEFAULT)
+ total_time = time.time() - start_time
+ result_code = 'OK'
+ label = OK_MESSAGE.format(total_time, port)
+ except:
+ result_code = 'CRITICAL'
+ label = CRITICAL_MESSAGE.format(host_name, port, traceback.format_exc())
+ except:
+ label = traceback.format_exc()
+ result_code = 'UNKNOWN'
+
+ return (result_code, [label])
+
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/kyuubi_client.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/kyuubi_client.py
new file mode 100755
index 0000000000..60d17141aa
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/kyuubi_client.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+# Python Imports
+import sys
+import os
+from resource_management import *
+
+
+class KyuubiClient(Script):
+ def install(self, env):
+ import params
+ self.install_packages(env)
+ self.configure(env)
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ Directory([params.kyuubi_log_dir, params.kyuubi_pid_dir, params.kyuubi_work_dir, params.kyuubi_metrics_dir, params.kyuubi_operation_log_dir],
+ #Directory([params.kyuubi_log_dir, params.kyuubi_pid_dir, params.kyuubi_operation_log_dir],
+ owner=params.kyuubi_user,
+ group=params.kyuubi_group,
+ mode=0o775,
+ create_parents = True
+ )
+
+ kyuubi_defaults = dict(params.config['configurations']['kyuubi-defaults'])
+
+ PropertiesFile(format("{kyuubi_conf_dir}/kyuubi-defaults.conf"),
+ properties = kyuubi_defaults,
+ key_value_delimiter = " ",
+ owner=params.kyuubi_user,
+ group=params.kyuubi_group,
+ mode=0o644
+ )
+
+ # create kyuubi-env.sh in kyuubi install dir
+ File(os.path.join(params.kyuubi_conf_dir, 'kyuubi-env.sh'),
+ owner=params.kyuubi_user,
+ group=params.kyuubi_group,
+ content=InlineTemplate(params.kyuubi_env_sh),
+ mode=0o644,
+ )
+
+
+
+
+if __name__ == "__main__":
+ KyuubiClient().execute()
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/kyuubi_server.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/kyuubi_server.py
new file mode 100755
index 0000000000..a3b63a0802
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/kyuubi_server.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+from resource_management import *
+
+class KyuubiServer(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None, config_dir=None):
+ import params
+ env.set_params(params)
+
+
+ Directory([params.kyuubi_log_dir, params.kyuubi_pid_dir, params.kyuubi_work_dir, params.kyuubi_metrics_dir, params.kyuubi_operation_log_dir],
+ owner=params.kyuubi_user,
+ group=params.kyuubi_group,
+ mode=0o775,
+ create_parents = True
+ )
+
+ Directory(params.kyuubi_home,
+ owner=params.kyuubi_user,
+ group=params.kyuubi_group,
+ mode=0o775
+ )
+
+ kyuubi_defaults = dict(params.config['configurations']['kyuubi-defaults'])
+
+ PropertiesFile(format("{kyuubi_conf_dir}/kyuubi-defaults.conf"),
+ properties = kyuubi_defaults,
+ key_value_delimiter = " ",
+ owner=params.kyuubi_user,
+ group=params.kyuubi_group,
+ mode=0o644
+ )
+
+ # create kyuubi-env.sh in kyuubi install dir
+ File(os.path.join(params.kyuubi_conf_dir, 'kyuubi-env.sh'),
+ owner=params.kyuubi_user,
+ group=params.kyuubi_group,
+ content=InlineTemplate(params.kyuubi_env_sh),
+ mode=0o644,
+ )
+
+ #create log4j2.properties kyuubi install dir
+ File(os.path.join(params.kyuubi_conf_dir, 'log4j2.properties'),
+ owner=params.kyuubi_user,
+ group=params.kyuubi_group,
+ content=InlineTemplate(params.kyuubi_log4j2_properties),
+ mode=0o644,
+ )
+ # work dir
+ Directory(os.path.join(params.kyuubi_pid_dir, 'work'),
+ owner=params.kyuubi_user,
+ group=params.kyuubi_group,
+ mode=0o775,
+ create_parents = True
+ )
+
+ # hdfs dir
+ params.HdfsResource(params.kyuubi_hdfs_user_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.kyuubi_user,
+ mode=0o775
+ )
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ self.configure(env)
+ Execute(params.kyuubi_start_cmd,user=params.kyuubi_user,environment={'JAVA_HOME': params.java_home})
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ Execute(params.kyuubi_stop_cmd,user=params.kyuubi_user,environment={'JAVA_HOME': params.java_home})
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+ check_process_status(params.kyuubi_pid_file)
+
+ def get_user(self):
+ import params
+ return params.kyuubi_user
+
+ def get_pid_files(self):
+ import params
+ return [params.kyuubi_pid_file]
+
+if __name__ == "__main__":
+ KyuubiServer().execute()
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/params.py
new file mode 100755
index 0000000000..45b1758095
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/params.py
@@ -0,0 +1,181 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import socket
+import os
+from urllib.parse import urlparse
+
+from ambari_commons.constants import AMBARI_SUDO_BINARY
+from resource_management import *
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.constants import StackFeature
+from resource_management.libraries.functions import conf_select, stack_select
+from resource_management.libraries.functions.version import format_stack_version, get_major_version
+from resource_management.libraries.functions.copy_tarball import get_sysprep_skip_copy_tarballs_hdfs
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.script.script import Script
+from ambari_commons.constants import AMBARI_SUDO_BINARY
+from resource_management.libraries.functions.copy_tarball import get_current_version
+from resource_management.libraries.functions.stack_features import check_stack_feature, get_stack_feature_version
+from resource_management.libraries.functions import StackFeature
+
+config = Script.get_config()
+
+java_home = config['ambariLevelParams']['java_home']
+stack_root = Script.get_stack_root()
+
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+sudo = AMBARI_SUDO_BINARY
+fqdn = socket.getfqdn().lower()
+
+retryAble = default("/commandParams/command_retry_enabled", False)
+
+cluster_name = config['clusterName']
+stack_name = default("/clusterLevelParams/stack_name", None)
+stack_root = Script.get_stack_root()
+#3.2
+stack_version_unformatted = config['clusterLevelParams']['stack_version']
+#3.2.0.0
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+#3.2
+major_stack_version = get_major_version(stack_version_formatted)
+
+#3.2.1.0-001
+effective_version = get_current_version(service="SPARK")
+
+sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs()
+
+# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade
+version = default("/commandParams/version", None)
+
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+
+if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+ hadoop_home = stack_select.get_hadoop_dir("home")
+
+
+component_directory = "kyuubi-server"
+kyuubi_home = format("{stack_root}/current/{component_directory}")
+kyuubi_conf_dir = format("{stack_root}/current/{component_directory}/conf")
+
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+user_group = config['configurations']['cluster-env']['user_group']
+
+kyuubi_user = config['configurations']['kyuubi-env']['kyuubi_user']
+kyuubi_group = config['configurations']['kyuubi-env']['kyuubi_group']
+kyuubi_pid_dir = config['configurations']['kyuubi-env']['kyuubi_pid_dir']
+kyuubi_log_dir = config['configurations']['kyuubi-env']['kyuubi_log_dir']
+kyuubi_work_dir = format("{kyuubi_home}/work")
+kyuubi_pid_file = format("{kyuubi_pid_dir}/kyuubi-{kyuubi_user}-org.apache.kyuubi.server.KyuubiServer.pid")
+
+kyuubi_metrics_dir= format("{stack_root}/current/{component_directory}/work/metrics")
+kyuubi_operation_log_dir= format("{stack_root}/current/{component_directory}/work/server_operation_logs")
+
+kyuubi_env_sh = config['configurations']['kyuubi-env']['content']
+kyuubi_log4j2_properties = config['configurations']['kyuubi-log4j2-properties']['content']
+
+kyuubi_hdfs_user_dir = format("/user/{kyuubi_user}")
+
+kyuubi_start_cmd = format("{kyuubi_home}/bin/kyuubi start")
+kyuubi_stop_cmd = format("{kyuubi_home}/bin/kyuubi stop")
+
+# zookeeper
+zookeeper_hosts = config['clusterHostInfo']['zookeeper_server_hosts']
+zookeeper_port = default('/configurations/zoo.cfg/clientPort', None)
+
+# get comma separated lists of zookeeper hosts from clusterHostInfo
+index = 0
+zookeeper_quorum = ""
+for host in zookeeper_hosts:
+ zookeeper_host = host
+ if zookeeper_port is not None:
+ zookeeper_host = host + ":" + str(zookeeper_port)
+
+ zookeeper_quorum += zookeeper_host
+ index += 1
+ if index < len(zookeeper_hosts):
+ zookeeper_quorum += ","
+
+cluster_zookeeper_quorum = zookeeper_quorum
+
+
+
+
+#kyuubi_server_port
+kyuubi_server_port = config['configurations']['kyuubi-defaults']['kyuubi.frontend.thrift.binary.bind.port']
+
+#kyuubi_client_home = format("{stack_root}/current/kyuubi-client")
+#kyuubi_client_conf_dir = format("{stack_root}/current/kyuubi-client/conf")
+
+smoke_user = config['configurations']['cluster-env']['smokeuser']
+
+kyuubi_server_hosts = default("/clusterHostInfo/kyuubi_server_hosts", [])
+has_kyuubi_server = not len(kyuubi_server_hosts) == 0
+
+
+kyuubi_authentication = "NONE"
+
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+kyuubi_kerberos_keytab = config['configurations']['kyuubi-defaults']['kyuubi.kinit.keytab']
+kyuubi_kerberos_principal = config['configurations']['kyuubi-defaults']['kyuubi.kinit.principal']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+hive_kerberos_keytab = config['configurations']['hive-site']['hive.server2.authentication.kerberos.keytab']
+
+security_enabled = default("/configurations/cluster-env/security_enabled", None)
+#kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+
+if security_enabled:
+ kyuubi_authentication = "KERBEROS"
+ hive_kerberos_principal = config['configurations']['hive-site']['hive.server2.authentication.kerberos.principal'].replace('_HOST', socket.getfqdn().lower())
+
+#for create_hdfs_directory
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+hdfs_site = config['configurations']['hdfs-site']
+hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore"
+
+dfs_type = default("/clusterLevelParams/dfs_type", "")
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file = hdfs_resource_ignore_file,
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs,
+ immutable_paths = get_not_managed_resources(),
+ dfs_type = dfs_type
+)
+
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/service_check.py
new file mode 100755
index 0000000000..cf7adfac8b
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/package/scripts/service_check.py
@@ -0,0 +1,67 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agree in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import subprocess
+import time
+import os
+
+from resource_management.core.exceptions import Fail
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.format import format
+from resource_management.core.resources.system import Execute
+from resource_management.core.logger import Logger
+
+
+CHECK_COMMAND_TIMEOUT_DEFAULT = 300.0
+
+class KyuubiServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ if params.security_enabled:
+ #kyuubi_kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; ")
+ kyuubi_kinit_cmd = format("{kinit_path_local} -kt {hive_kerberos_keytab} {hive_kerberos_principal}; ")
+ Execute(kyuubi_kinit_cmd, user=params.smoke_user)
+
+
+ if params.has_kyuubi_server:
+ healthy_kyuubi_host = ""
+ for kyuubi_host in params.kyuubi_server_hosts:
+ if params.security_enabled:
+ kerberos_principal = params.kyuubi_kerberos_principal.replace('_HOST', kyuubi_host)
+ beeline_url = ["jdbc:hive2://{kyuubi_host}:{kyuubi_server_port}/default;principal={kerberos_principal}"]
+ else:
+ beeline_url = ["jdbc:hive2://{kyuubi_host}:{kyuubi_server_port}/default"]
+
+ beeline_cmd = os.path.join(params.kyuubi_home, "bin", "beeline")
+ cmd = "! %s -u '%s' -n hive -e '' 2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL' -e 'Error: Could not open'" % \
+ (beeline_cmd, format(";".join(beeline_url)))
+
+ try:
+ Execute(cmd, user=params.smoke_user, path=[beeline_cmd], timeout=CHECK_COMMAND_TIMEOUT_DEFAULT)
+ healthy_kyuubi_host = kyuubi_host
+ break
+ except:
+ pass
+
+ if len(params.kyuubi_server_hosts) > 0 and healthy_kyuubi_host == "":
+ raise Fail("Connection to all Kyuubi servers failed.")
+
+if __name__ == "__main__":
+ KyuubiServiceCheck().execute()
+
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/quicklinks/quicklinks.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/quicklinks/quicklinks.json
new file mode 100755
index 0000000000..c56b74e064
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/quicklinks/quicklinks.json
@@ -0,0 +1,21 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "links": [
+ {
+ "name": "Kyuubi_WEB_UI",
+ "label": "Kyuubi Web UI",
+ "requires_user_name": "false",
+ "component_name": "KYUUBI_SERVER",
+ "url":"%@://%@:%@",
+ "port":{
+ "http_property": "kyuubi.frontend.rest.bind.port",
+ "http_default_port": "10099",
+ "regex": "^(\\d+)$",
+ "site": "kyuubi-defaults"
+ }
+ }
+ ]
+ }
+}
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/role_command_order.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/role_command_order.json
new file mode 100755
index 0000000000..f54046c06a
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/role_command_order.json
@@ -0,0 +1,7 @@
+{
+ "general_deps" : {
+ "_comment" : "dependencies for KYUUBI",
+ "KYUUBI_SERVICE_CHECK-SERVICE_CHECK" : ["KYUUBI_SERVER-START"],
+ "KYUUBI_SERVER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START","SECONDARY_NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START", "HIVE_METASTORE-START", "SPARK_JOBHISTORYSERVER-START"]
+ }
+}
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/service_advisor.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/service_advisor.py
new file mode 100755
index 0000000000..429981fe0c
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/KYUUBI/service_advisor.py
@@ -0,0 +1,206 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# Python imports
+from ast import Param
+import imp
+import os
+import traceback
+import re
+import socket
+import fnmatch
+
+
+from resource_management.core.logger import Logger
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+ if "BASE_SERVICE_ADVISOR" in os.environ:
+ PARENT_FILE = os.environ["BASE_SERVICE_ADVISOR"]
+ with open(PARENT_FILE, 'rb') as fp:
+ service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print "Failed to load parent"
+
+
+DB_TYPE_DEFAULT_PORT_MAP = {"MYSQL":"3306", "ORACLE":"1521", "POSTGRES":"5432", "MSSQL":"1433", "SQLA":"2638"}
+
+class KyuubiServiceAdvisor(service_advisor.ServiceAdvisor):
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(KyuubiServiceAdvisor, self)
+ self.as_super.__init__(*args, **kwargs)
+
+ # Always call these methods
+ self.modifyMastersWithMultipleInstances()
+ self.modifyCardinalitiesDict()
+ self.modifyHeapSizeProperties()
+ self.modifyNotValuableComponents()
+ self.modifyComponentsNotPreferableOnServer()
+ self.modifyComponentLayoutSchemes()
+
+ def modifyMastersWithMultipleInstances(self):
+ """
+ Modify the set of masters with multiple instances.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyCardinalitiesDict(self):
+ """
+ Modify the dictionary of cardinalities.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyHeapSizeProperties(self):
+ """
+ Modify the dictionary of heap size properties.
+ Must be overriden in child class.
+ """
+ pass
+
+ def modifyNotValuableComponents(self):
+ """
+ Modify the set of components whose host assignment is based on other services.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyComponentsNotPreferableOnServer(self):
+ """
+ Modify the set of components that are not preferable on the server.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyComponentLayoutSchemes(self):
+ """
+ Modify layout scheme dictionaries for components.
+ The scheme dictionary basically maps the number of hosts to
+ host index where component should exist.
+ Must be overriden in child class.
+ """
+
+ # Nothing to do
+ pass
+
+
+ def getServiceComponentLayoutValidations(self, services, hosts):
+ """
+ Get a list of errors.
+ Must be overriden in child class.
+ """
+
+ return self.getServiceComponentCardinalityValidations(services, hosts, "KYUUBI")
+
+ def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts):
+ """
+ Entry point.
+ Must be overriden in child class.
+ """
+ # Logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
+ # (self.__class__.__name__, inspect.stack()[0][3]))
+
+ recommender = KyuubiRecommender()
+ recommender.recommendKyuubiConfigurationsFromHDP33(configurations, clusterData, services, hosts)
+
+
+ # def getServiceConfigurationRecommendationsForSSO(self, configurations, clusterData, services, hosts):
+ # """
+ # Entry point.
+ # Must be overriden in child class.
+ # """
+ # recommender = KyuubiRecommender()
+ # recommender.recommendConfigurationsForSSO(configurations, clusterData, services, hosts)
+
+ def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
+ """
+ Entry point.
+ Validate configurations for the service. Return a list of errors.
+ The code for this function should be the same for each Service Advisor.
+ """
+ # Logger.info("Class: %s, Method: %s. Validating Configurations." %
+ # (self.__class__.__name__, inspect.stack()[0][3]))
+
+ return []
+
+ @staticmethod
+ def isKerberosEnabled(services, configurations):
+ """
+ Determines if security is enabled by testing the value of core-site/hadoop.security.authentication enabled.
+ If the property exists and is equal to "kerberos", then is it enabled; otherwise is it assumed to be
+ disabled.
+
+ :type services: dict
+ :param services: the dictionary containing the existing configuration values
+ :type configurations: dict
+ :param configurations: the dictionary containing the updated configuration values
+ :rtype: bool
+ :return: True or False
+ """
+ if configurations and "core-site" in configurations and \
+ "hadoop.security.authentication" in configurations["core-site"]["properties"]:
+ return configurations["core-site"]["properties"]["hadoop.security.authentication"].lower() == "kerberos"
+ elif services and "core-site" in services["configurations"] and \
+ "hadoop.security.authentication" in services["configurations"]["core-site"]["properties"]:
+ return services["configurations"]["core-site"]["properties"]["hadoop.security.authentication"].lower() == "kerberos"
+ else:
+ return False
+
+
+class KyuubiRecommender(service_advisor.ServiceAdvisor):
+ """
+ Kyuubi Recommender suggests properties when adding the service for the first time or modifying configs via the UI.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(KyuubiRecommender, self)
+ self.as_super.__init__(*args, **kwargs)
+
+ def recommendKyuubiConfigurationsFromHDP33(self, configurations, clusterData, services, hosts):
+ """
+ Recommend configurations for this service based on HDP 3.3.
+ """
+
+ putKyuubiDefaultsProperty = self.putProperty(configurations, "kyuubi-defaults", services)
+ putKyuubiDefaultsPropertyAttribute = self.putPropertyAttribute(configurations, "kyuubi-defaults")
+
+ zookeeper_host_port = self.getZKHostPortString(services)
+ putKyuubiDefaultsProperty("kyuubi.ha.zookeeper.quorum", zookeeper_host_port)
+ putKyuubiDefaultsProperty("kyuubi.ha.addresses", zookeeper_host_port)
+ putKyuubiDefaultsProperty("kyuubi.ha.client.class", 'org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient')
+
+
+
+
+
+
+
+
+
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ambari.apache.org
For additional commands, e-mail: commits-help@ambari.apache.org