You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2017/02/09 20:32:46 UTC
ambari git commit: AMBARI-19870. Add Superset as a UI for Druid in
HDP (Nishant Bangarwa via smohanty)
Repository: ambari
Updated Branches:
refs/heads/trunk d5755ba15 -> 3c0f3c46b
AMBARI-19870. Add Superset as a UI for Druid in HDP (Nishant Bangarwa via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3c0f3c46
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3c0f3c46
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3c0f3c46
Branch: refs/heads/trunk
Commit: 3c0f3c46b97f00a24cf0711c43bc24df3ed6df84
Parents: d5755ba
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Feb 9 12:32:17 2017 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Feb 9 12:32:17 2017 -0800
----------------------------------------------------------------------
.../DRUID/0.9.2/configuration/druid-common.xml | 10 +-
.../0.9.2/configuration/druid-superset-env.xml | 115 ++++++++++++
.../0.9.2/configuration/druid-superset.xml | 178 +++++++++++++++++++
.../common-services/DRUID/0.9.2/metainfo.xml | 26 ++-
.../DRUID/0.9.2/package/scripts/params.py | 45 ++++-
.../0.9.2/package/scripts/status_params.py | 2 +
.../DRUID/0.9.2/package/scripts/superset.py | 153 ++++++++++++++++
.../DRUID/0.9.2/package/templates/superset.sh | 95 ++++++++++
.../DRUID/0.9.2/quicklinks/quicklinks.json | 13 ++
.../DRUID/0.9.2/role_command_order.json | 18 ++
.../DRUID/0.9.2/themes/theme.json | 84 ++++++++-
.../stacks/HDP/2.6/role_command_order.json | 19 --
.../stacks/HDP/2.6/services/DRUID/kerberos.json | 33 ++++
.../stacks/HDP/2.6/services/stack_advisor.py | 11 ++
.../test/python/stacks/2.6/configs/default.json | 3 +
15 files changed, 774 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml
index a494750..d3b53cd 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-common.xml
@@ -132,20 +132,20 @@
<property>
<name>database_name</name>
<value>druid</value>
- <display-name>Metadata storage database name</display-name>
- <description>Metadata storage database name</description>
+ <display-name>Druid Metadata storage database name</display-name>
+ <description>Druid Metadata storage database name</description>
<on-ambari-upgrade add="false"/>
</property>
<property>
<name>metastore_hostname</name>
<value>localhost</value>
- <display-name>Metadata storage hostname name</display-name>
- <description>Metadata storage hostname name</description>
+ <display-name>Metadata storage hostname</display-name>
+ <description>Metadata storage hostname</description>
<on-ambari-upgrade add="false"/>
</property>
<property require-input="true">
<name>druid.metadata.storage.type</name>
- <display-name>Metadata storage type</display-name>
+ <display-name>Druid Metadata storage type</display-name>
<value>derby</value>
<value-attributes>
<overridable>false</overridable>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset-env.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset-env.xml
new file mode 100644
index 0000000..61421c0
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset-env.xml
@@ -0,0 +1,115 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>superset_log_dir</name>
+ <value>/var/log/superset</value>
+ <display-name>Superset log dir</display-name>
+ <description>Superset log directory.</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>superset_pid_dir</name>
+ <value>/var/run/superset</value>
+ <display-name>Superset pid dir</display-name>
+ <description>Superset pid directory.</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>superset_admin_user</name>
+ <value>admin</value>
+ <display-name>Superset Admin Username</display-name>
+ <description>
+ Superset Username. This value cannot be modified by Ambari
+ except on initial install. Please make sure the username change in
+ Superset is reflected in Ambari.
+ </description>
+ <value-attributes>
+ <type>db_user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property require-input="true">
+ <name>superset_admin_password</name>
+ <value/>
+ <property-type>PASSWORD</property-type>
+ <display-name>Superset Admin Password</display-name>
+ <description>
+ Superset password. This value cannot be modified by Ambari
+ except on initial install. Please make sure the password change in
+ Superset is reflected back in Ambari.
+ </description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>superset_admin_firstname</name>
+ <value></value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>superset_admin_lastname</name>
+ <value></value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>superset_admin_email</name>
+ <value></value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>content</name>
+ <display-name>superset-env template</display-name>
+ <value>
+ # Set environment variables here.
+
+ # Superset Home Dir
+ export SUPERSET_CONFIG_DIR={{superset_config_dir}}
+
+ # Superset Log Dir
+ export SUPERSET_LOG_DIR={{superset_log_dir}}
+
+ # Superset Log Dir
+ export SUPERSET_BIN_DIR={{superset_bin_dir}}
+
+ # Superset PID Dir
+ export SUPERSET_PID_DIR={{superset_pid_dir}}
+
+ # Add superset_config.py to PYTHONPATH
+ export PYTHONPATH=$PYTHONPATH:{{superset_config_dir}}
+
+ export SUPERSET_WEBSERVER_PORT={{superset_webserver_port}}
+ export SUPERSET_WEBSERVER_ADDRESS={{superset_webserver_address}}
+ export SUPERSET_TIMEOUT={{superset_timeout}}
+ export SUPERSET_WORKERS={{superset_workers}}
+ </value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset.xml
new file mode 100644
index 0000000..2fff10a
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-superset.xml
@@ -0,0 +1,178 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>ROW_LIMIT</name>
+ <value>5000</value>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <description></description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>SUPERSET_WORKERS</name>
+ <value>4</value>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <description></description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>SUPERSET_WEBSERVER_PORT</name>
+ <value>9088</value>
+ <description></description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>SUPERSET_WEBSERVER_ADDRESS</name>
+ <value>0.0.0.0</value>
+ <description></description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>SUPERSET_TIMEOUT</name>
+ <value>60</value>
+ <description></description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property require-input="true">
+ <name>SUPERSET_DATABASE_TYPE</name>
+ <display-name>Superset Database type</display-name>
+ <value>sqlite</value>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>mysql</value>
+ <label>MYSQL</label>
+ </entry>
+ <entry>
+ <value>sqlite</value>
+ <label>SQLITE</label>
+ </entry>
+ <entry>
+ <value>postgresql</value>
+ <label>POSTGRESQL</label>
+ </entry>
+ </entries>
+ </value-attributes>
+ <description>Type of the Database. Note that sqlite db will be installed on the node where superset is installed.
+ Use mysql or postgred when installing superset on multiple nodes.
+ mysql installed by ambari is only for development and not suitable for production use cases due to it being not HA.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>SUPERSET_DATABASE_NAME</name>
+ <value>superset</value>
+ <display-name>Superset Database name</display-name>
+ <description>Superset Database name</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>SUPERSET_DATABASE_PASSWORD</name>
+ <value></value>
+ <property-type>PASSWORD</property-type>
+ <display-name>Superset Database password</display-name>
+ <description>Password for the database.</description>
+ <value-attributes>
+ <type>password</type>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>SUPERSET_DATABASE_USER</name>
+ <value>superset</value>
+ <display-name>Superset Database user</display-name>
+ <description>Superset Database user</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>SUPERSET_DATABASE_PORT</name>
+ <value></value>
+ <display-name>Superset Database port</display-name>
+ <description>Superset Database port</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>superset</type>
+ <name>SUPERSET_DATABASE_TYPE</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>SUPERSET_DATABASE_NAME</name>
+ <value>superset</value>
+ <display-name>Superset Database name</display-name>
+ <description>Superset Database name</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>SUPERSET_DATABASE_HOSTNAME</name>
+ <value>localhost</value>
+ <display-name>Database hostname</display-name>
+ <description>Database hostname</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>WTF_CSRF_ENABLED</name>
+ <value>True</value>
+ <description></description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ENABLE_PROXY_FIX</name>
+ <value>True</value>
+ <description>When using Superset beind a Proxy server set this to True.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>MAPBOX_API_KEY</name>
+ <value></value>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <description>API key to enable Mapbox visualizations</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>SECRET_KEY</name>
+ <value></value>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <description>Secret Key used to encrypt user passwords. A Long Random String is recommended.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml
index c837bbb..c0859f2 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/metainfo.xml
@@ -158,10 +158,24 @@
<scriptType>PYTHON</scriptType>
<timeout>600</timeout>
</commandScript>
+ <configuration-dependencies>
+ <config-type>druid-router</config-type>
+ </configuration-dependencies>
+ </component>
+ <component>
+ <name>DRUID_SUPERSET</name>
+ <displayName>Druid Superset</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/superset.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ <configuration-dependencies>
+ <config-type>druid-superset</config-type>
+ </configuration-dependencies>
</component>
- <configuration-dependencies>
- <config-type>druid-router</config-type>
- </configuration-dependencies>
</components>
<themes>
@@ -178,6 +192,9 @@
<package>
<name>druid_${stack_version}</name>
</package>
+ <package>
+ <name>superset_${stack_version}</name>
+ </package>
</packages>
</osSpecific>
<osSpecific>
@@ -186,6 +203,9 @@
<package>
<name>druid-${stack_version}</name>
</package>
+ <package>
+ <name>superset-${stack_version}</name>
+ </package>
</packages>
</osSpecific>
</osSpecifics>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
index aed4043..61f35b2 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
@@ -149,7 +149,6 @@ HdfsResource = functools.partial(
dfs_type=dfs_type
)
-
# Ambari Metrics
metric_emitter_type = "noop"
metric_collector_host = ""
@@ -184,3 +183,47 @@ if has_metric_collector:
metric_collector_protocol = 'http'
pass
+superset_home_dir = format("{stack_root}/current/druid-superset")
+superset_bin_dir = format("{superset_home_dir}/bin")
+superset_log_dir = default("/configurations/druid-superset-env/superset_log_dir", '/var/log/superset')
+superset_pid_dir = status_params.superset_pid_dir
+superset_config_dir = '/etc/superset/conf'
+superset_admin_user = config['configurations']['druid-superset-env']['superset_admin_user']
+superset_admin_password = config['configurations']['druid-superset-env']['superset_admin_password']
+superset_admin_firstname = config['configurations']['druid-superset-env']['superset_admin_firstname']
+superset_admin_lastname = config['configurations']['druid-superset-env']['superset_admin_lastname']
+superset_admin_email = config['configurations']['druid-superset-env']['superset_admin_email']
+superset_env_sh_template = config['configurations']['druid-superset-env']['content']
+superset_protocol = "http"
+superset_webserver_address=config['configurations']['druid-superset']['SUPERSET_WEBSERVER_ADDRESS']
+superset_webserver_port = config['configurations']['druid-superset']['SUPERSET_WEBSERVER_PORT']
+superset_timeout = config['configurations']['druid-superset']['SUPERSET_TIMEOUT']
+superset_workers = config['configurations']['druid-superset']['SUPERSET_WORKERS']
+superset_hosts = default('/clusterHostInfo/superset_hosts', None)
+
+# superset database configs
+superset_db_type = config['configurations']['druid-superset']['SUPERSET_DATABASE_TYPE']
+superset_db_name = config['configurations']['druid-superset']['SUPERSET_DATABASE_NAME']
+superset_db_password = config['configurations']['druid-superset']['SUPERSET_DATABASE_PASSWORD']
+superset_db_user = config['configurations']['druid-superset']['SUPERSET_DATABASE_USER']
+superset_db_port = config['configurations']['druid-superset']['SUPERSET_DATABASE_PORT']
+superset_db_host = config['configurations']['druid-superset']['SUPERSET_DATABASE_HOSTNAME']
+
+superset_db_uri = None
+if superset_db_type == "sqlite":
+ superset_db_uri = format("sqlite:///{superset_config_dir}/{superset_db_name}.db")
+elif superset_db_type == "postgresql":
+ superset_db_uri = format("postgresql+pygresql://{superset_db_user}:{superset_db_password}@{superset_db_host}:{superset_db_port}/{superset_db_name}")
+elif superset_db_type == "mysql":
+ superset_db_uri = format("mysql+pymysql://{superset_db_user}:{superset_db_password}@{superset_db_host}:{superset_db_port}/{superset_db_name}")
+
+druid_coordinator_hosts = default("/clusterHostInfo/druid_coordinator_hosts", [])
+druid_coordinator_host = ""
+if not len(druid_coordinator_hosts) == 0:
+ druid_coordinator_host = druid_coordinator_hosts[0]
+druid_router_hosts = default("/clusterHostInfo/druid_coordinator_hosts", [])
+druid_router_host = ""
+if not len(druid_router_hosts) == 0:
+ druid_router_host = druid_router_hosts[0]
+druid_coordinator_port = config['configurations']['druid-coordinator']['druid.port']
+druid_router_port = config['configurations']['druid-router']['druid.port']
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
index ee1d61c..d48ff83 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
@@ -22,3 +22,5 @@ from resource_management.libraries.script.script import Script
config = Script.get_config()
druid_pid_dir = config['configurations']['druid-env']['druid_pid_dir']
+superset_pid_dir = config['configurations']['druid-superset-env']['superset_pid_dir']
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py
new file mode 100644
index 0000000..6a56e25
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/superset.py
@@ -0,0 +1,153 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os, time
+from resource_management.core.resources.system import Directory
+from resource_management import Script
+from resource_management.libraries.resources.properties_file import PropertiesFile
+from resource_management.core.resources.system import Execute
+from resource_management.core.source import Template
+from resource_management.libraries.functions.show_logs import show_logs
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.core.resources import File
+
+class Superset(Script):
+
+ def get_component_name(self):
+ return format("druid-superset")
+
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ Directory(
+ [params.superset_pid_dir, params.superset_log_dir, params.superset_config_dir, params.superset_home_dir],
+ mode=0755,
+ cd_access='a',
+ owner=params.druid_user,
+ group=params.user_group,
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ File(format("{params.superset_config_dir}/superset-env.sh"),
+ mode=0755,
+ owner=params.druid_user,
+ group=params.user_group,
+ content=InlineTemplate(params.superset_env_sh_template)
+ )
+
+ File(os.path.join(params.superset_bin_dir, 'superset.sh'),
+ owner=params.druid_user,
+ group=params.user_group,
+ mode=0755,
+ content=Template("superset.sh")
+ )
+ superset_config = mutable_config_dict(params.config["configurations"]["druid-superset"])
+
+ if params.superset_db_uri:
+ superset_config["SQLALCHEMY_DATABASE_URI"] = params.superset_db_uri
+
+ PropertiesFile("superset_config.py",
+ dir=params.superset_config_dir,
+ properties=quote_string_values(superset_config),
+ owner=params.druid_user,
+ group=params.user_group
+ )
+
+ # Initialize DB and create admin user.
+ Execute(format("source {params.superset_config_dir}/superset-env.sh ; {params.superset_bin_dir}/superset db upgrade"),
+ user=params.druid_user)
+ Execute(format("source {params.superset_config_dir}/superset-env.sh ; {params.superset_bin_dir}/fabmanager create-admin --app superset --username {params.superset_admin_user} --password {params.superset_admin_password} --firstname {params.superset_admin_firstname} --lastname {params.superset_admin_lastname} --email {params.superset_admin_email}"),
+ user=params.druid_user)
+ Execute(format("source {params.superset_config_dir}/superset-env.sh ; {params.superset_bin_dir}/superset init"),
+ user=params.druid_user)
+
+ # Configure Druid Cluster in superset DB
+ Execute(format("source {params.superset_config_dir}/superset-env.sh ; {params.superset_bin_dir}/superset configure_druid_cluster --name druid-ambari --coordinator-host {params.druid_coordinator_host} --coordinator-port {params.druid_coordinator_port} --broker-host {params.druid_router_host} --broker-port {params.druid_router_port} --coordinator-endpoint druid/coordinator/v1/metadata --broker-endpoint druid/v2"),
+ user=params.druid_user)
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ daemon_cmd = self.get_daemon_cmd(params, "start")
+ try:
+ Execute(daemon_cmd,
+ user=params.druid_user
+ )
+ except:
+ show_logs(params.druid_log_dir, params.druid_user)
+ raise
+
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ daemon_cmd = self.get_daemon_cmd(params, "stop")
+ try:
+ Execute(daemon_cmd,
+ user=params.druid_user
+ )
+ except:
+ show_logs(params.druid_log_dir, params.druid_user)
+ raise
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = status_params.superset_pid_dir + '/superset.pid'
+ check_process_status(pid_file)
+
+ def get_log_folder(self):
+ import params
+ return params.druid_log_dir
+
+ def get_user(self):
+ import params
+ return params.druid_user
+
+ def get_daemon_cmd(self, params=None, command=None):
+ return format('source {params.superset_config_dir}/superset-env.sh ; {params.superset_bin_dir}/superset.sh {command}')
+
+def mutable_config_dict(config):
+ rv = {}
+ for key, value in config.iteritems():
+ rv[key] = value
+ return rv
+
+def quote_string_values(config):
+ rv = {}
+ for key, value in config.iteritems():
+ rv[key] = quote_string_value(value)
+ return rv
+
+def quote_string_value(value):
+ if value.lower() == "true" or value.lower() == "false" or value.isdigit():
+ return value
+ else:
+ return "'{}'".format(value)
+
+
+
+if __name__ == "__main__":
+ Superset().execute()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/templates/superset.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/templates/superset.sh b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/templates/superset.sh
new file mode 100644
index 0000000..3e327a6
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/templates/superset.sh
@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+## Runs superset as a daemon
+## Environment Variables used by this script -
+## SUPERSET_CONFIG_DIR - directory having druid config files
+## SUPERSET_LOG_DIR - directory used to store superset logs
+## SUPERSET_PID_DIR - directory used to store pid file
+
+usage="Usage: superset.sh (start|stop|status)"
+
+if [ $# -le 0 ]; then
+ echo $usage
+ exit 1
+fi
+
+command=$1
+
+CONF_DIR="${SUPERSET_CONFIG_DIR:=/etc/superset/conf}"
+LOG_DIR="${SUPERSET_LOG_DIR:=/var/log/superset}"
+PID_DIR="${SUPERSET_PID_DIR:=/var/run/superset}"
+TIMEOUT="${SUPERSET_TIMEOUT:=60}"
+WEBSERVER_ADDRESS="${SUPERSET_WEBSERVER_ADDRESS:=0.0.0.0}"
+WEBSERVER_PORT="${SUPERSET_WEBSERVER_PORT:=9088}"
+WORKERS="${SUPERSET_WORKERS:=4}"
+BIN_DIR="${SUPERSET_BIN_DIR}"
+
+pid=$PID_DIR/superset.pid
+
+case $command in
+ (start)
+
+ if [ -f $pid ]; then
+ if kill -0 `cat $pid| head -n 1` > /dev/null 2>&1; then
+ echo Superset node running as process `cat $pid | head -n 1`. Stop it first.
+ exit 1
+ fi
+ fi
+
+ $BIN_DIR/gunicorn -D --workers $WORKERS -p $pid --log-file $LOG_DIR/superset.log -t $TIMEOUT -b $WEBSERVER_ADDRESS:$WEBSERVER_PORT --limit-request-line 0 --limit-request-field_size 0 superset:app
+
+ echo "Started Superset"
+ ;;
+
+ (stop)
+
+ if [ -f $pid ]; then
+ TARGET_PID=`cat $pid | head -n 1`
+ if kill -0 $TARGET_PID > /dev/null 2>&1; then
+ echo Stopping process `cat $pid | head -n 1`...
+ kill $TARGET_PID
+ else
+ echo No superset node to stop
+ fi
+ rm -f $pid
+ else
+ echo No superset node to stop
+ fi
+ ;;
+
+ (status)
+ if [ -f $pid ]; then
+ if kill -0 `cat $pid | head -n 1` > /dev/null 2>&1; then
+ echo RUNNING
+ exit 0
+ else
+ echo STOPPED
+ fi
+ else
+ echo STOPPED
+ fi
+ ;;
+
+ (*)
+ echo $usage
+ exit 1
+ ;;
+esac
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
index c68b9b9..16f5d5c 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
@@ -31,6 +31,19 @@
"regex": "^(\\d+)$",
"site": "druid-overlord"
}
+ },
+ {
+ "name": "superset",
+ "label": "Superset",
+ "component_name": "DRUID_SUPERSET",
+ "requires_user_name": "false",
+ "url": "%@://%@:%@",
+ "port": {
+ "http_property": "SUPERSET_WEBSERVER_PORT",
+ "http_default_port": "9088",
+ "regex": "^(\\d+)$",
+ "site": "druid-superset"
+ }
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json
new file mode 100644
index 0000000..8f65c2e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json
@@ -0,0 +1,18 @@
+{
+ "general_deps" : {
+ "_comment" : "dependencies for Druid",
+ "DRUID_HISTORICAL-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "DRUID_OVERLORD-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "DRUID_MIDDLEMANAGER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "DRUID_BROKER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "DRUID_ROUTER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "DRUID_COORDINATOR-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "DRUID_SUPERSET-START" : ["DRUID_COORDINATOR-START", "DRUID_BROKER-START"],
+ "DRUID_OVERLORD-RESTART" : ["DRUID_HISTORICAL-RESTART"],
+ "DRUID_MIDDLEMANAGER-RESTART" : ["DRUID_OVERLORD-RESTART"],
+ "DRUID_BROKER-RESTART" : ["DRUID_MIDDLEMANAGER-RESTART"],
+ "DRUID_ROUTER-RESTART" : ["DRUID_BROKER-RESTART"],
+ "DRUID_COORDINATOR-RESTART" : ["DRUID_ROUTER-RESTART"],
+ "DRUID_SERVICE_CHECK-SERVICE_CHECK" : ["DRUID_HISTORICAL-START", "DRUID_COORDINATOR-START", "DRUID_OVERLORD-START", "DRUID_MIDDLEMANAGER-START", "DRUID_BROKER-START", "DRUID_ROUTER-START", "DRUID_SUPERSET-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
index f494594..151478b 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
@@ -10,7 +10,7 @@
"name": "metadata_storage",
"display-name": "META DATA STORAGE CONFIG",
"layout": {
- "tab-columns": "1",
+ "tab-columns": "2",
"tab-rows": "1",
"sections": [
{
@@ -19,17 +19,25 @@
"row-index": "0",
"column-index": "0",
"row-span": "2",
- "column-span": "1",
+ "column-span": "2",
"section-columns": "1",
"section-rows": "1",
"subsections": [
{
"name": "subsection-metadata-storage-row1-col1",
- "display-name": "META DATA STORAGE",
+ "display-name": "DRUID META DATA STORAGE",
"row-index": "0",
"column-index": "0",
"row-span": "1",
"column-span": "1"
+ },
+ {
+ "name": "subsection-metadata-storage-row1-col2",
+ "display-name": "SUPERSET META DATA STORAGE",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1"
}
]
}
@@ -69,6 +77,34 @@
{
"config": "druid-common/druid.metadata.storage.connector.connectURI",
"subsection-name": "subsection-metadata-storage-row1-col1"
+ },
+ {
+ "config": "druid-superset/SUPERSET_DATABASE_NAME",
+ "subsection-name": "subsection-metadata-storage-row1-col2"
+ },
+ {
+ "config": "druid-superset/SUPERSET_DATABASE_TYPE",
+ "subsection-name": "subsection-metadata-storage-row1-col2"
+ },
+ {
+ "config": "druid-superset/SUPERSET_DATABASE_USER",
+ "subsection-name": "subsection-metadata-storage-row1-col2"
+ },
+ {
+ "config": "druid-superset/SUPERSET_DATABASE_PASSWORD",
+ "subsection-name": "subsection-metadata-storage-row1-col2"
+ },
+ {
+ "config": "druid-superset/SUPERSET_DATABASE_HOSTNAME",
+ "subsection-name": "subsection-metadata-storage-row1-col2"
+ },
+ {
+ "config": "druid-superset/SUPERSET_DATABASE_PORT",
+ "subsection-name": "subsection-metadata-storage-row1-col2"
+ },
+ {
+ "config": "druid-superset/SECRET_KEY",
+ "subsection-name": "subsection-metadata-storage-row1-col2"
}
]
},
@@ -114,6 +150,48 @@
"widget": {
"type": "text-field"
}
+ },
+ {
+ "config": "druid-superset/SUPERSET_DATABASE_NAME",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "druid-superset/SUPERSET_DATABASE_TYPE",
+ "widget": {
+ "type": "combo"
+ }
+ },
+ {
+ "config": "druid-superset/SUPERSET_DATABASE_USER",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "druid-superset/SUPERSET_DATABASE_PASSWORD",
+ "widget": {
+ "type": "password"
+ }
+ },
+ {
+ "config": "druid-superset/SUPERSET_DATABASE_HOSTNAME",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "druid-superset/SUPERSET_DATABASE_PORT",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "druid-superset/SECRET_KEY",
+ "widget": {
+ "type": "password"
+ }
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json
deleted file mode 100644
index 66b6ed2..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "_comment" : "Record format:",
- "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
- "general_deps" : {
- "_comment" : "dependencies for all cases",
- "DRUID_HISTORICAL-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
- "DRUID_OVERLORD-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
- "DRUID_MIDDLEMANAGER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
- "DRUID_BROKER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
- "DRUID_ROUTER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
- "DRUID_COORDINATOR-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
- "DRUID_OVERLORD-RESTART" : ["DRUID_HISTORICAL-RESTART"],
- "DRUID_MIDDLEMANAGER-RESTART" : ["DRUID_OVERLORD-RESTART"],
- "DRUID_BROKER-RESTART" : ["DRUID_MIDDLEMANAGER-RESTART"],
- "DRUID_ROUTER-RESTART" : ["DRUID_BROKER-RESTART"],
- "DRUID_COORDINATOR-RESTART" : ["DRUID_ROUTER-RESTART"],
- "DRUID_SERVICE_CHECK-SERVICE_CHECK" : ["DRUID_HISTORICAL-START", "DRUID_COORDINATOR-START", "DRUID_OVERLORD-START", "DRUID_MIDDLEMANAGER-START", "DRUID_BROKER-START", "DRUID_ROUTER-START"]
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json
index 251975b..0ba5071 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json
@@ -34,6 +34,27 @@
}
},
{
+ "name": "superset",
+ "principal": {
+ "value": "${druid-env/druid_user}@${realm}",
+ "type": "user",
+ "configuration": "druid-superset/KERBEROS_PRINCIPAL",
+ "local_username": "${druid-env/druid_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/superset.headless.keytab",
+ "owner": {
+ "name": "${druid-env/druid_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": "r"
+ },
+ "configuration": "druid-superset/KERBEROS_KEYTAB"
+ }
+ },
+ {
"name": "/smokeuser"
}
],
@@ -77,6 +98,14 @@
"name": "/druid"
}
]
+ },
+ {
+ "name": "DRUID_SUPERSET",
+ "identities": [
+ {
+ "name": "/druid"
+ }
+ ]
}
],
"configurations": [
@@ -84,6 +113,10 @@
"druid-common": {
"druid.hadoop.security.spnego.excludedPaths": "[\"/status\"]",
"druid.security.extensions.loadList" : "[\"druid-kerberos\"]"
+ },
+ "druid-superset": {
+ "ENABLE_KERBEROS_AUTHENTICATION" : "True",
+ "KERBEROS_REINIT_TIME_SEC" : 3600
}
}
],
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 957d625..e6ebd6f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -127,6 +127,17 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
putComponentProperty('druid.processing.numThreads', processingThreads)
putComponentProperty('druid.server.http.numThreads', max(10, (totalAvailableCpu * 17) / 16 + 2) + 30)
+ # superset is in list of services to be installed
+ if 'druid-superset' in services['configurations']:
+ # Recommendations for Superset
+ superset_database_type = services['configurations']["druid-superset"]["properties"]["SUPERSET_DATABASE_TYPE"]
+ putSupersetProperty = self.putProperty(configurations, "druid-superset", services)
+
+ if superset_database_type == "mysql":
+ putSupersetProperty("SUPERSET_DATABASE_PORT", "3306")
+ elif superset_database_type == "postgresql":
+ putSupersetProperty("SUPERSET_DATABASE_PORT", "5432")
+
def getMetadataConnectionString(self, database_type):
driverDict = {
'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true',
http://git-wip-us.apache.org/repos/asf/ambari/blob/3c0f3c46/ambari-server/src/test/python/stacks/2.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/configs/default.json b/ambari-server/src/test/python/stacks/2.6/configs/default.json
index 4d9f98c..a1f89ce 100644
--- a/ambari-server/src/test/python/stacks/2.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.6/configs/default.json
@@ -459,6 +459,9 @@
},
"druid-logrotate" : {
"content" : "<![CDATA[\n {{druid_log_dir}}/*.log {\n copytruncate\n rotate 7\n daily\n nocompress\n missingok\n notifempty\n create 660 druid users\n dateext\n dateformat -%Y-%m-%d-%s\n }\n ]]>\n"
+ },
+ "druid-superset" : {
+ "SUPERSET_DATABASE_TYPE" : "sqllite"
}
},
"configuration_attributes": {