You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by lm...@apache.org on 2017/10/26 14:22:47 UTC
[11/37] knox git commit: KNOX-1014 - Service Discovery and Topology
Generation Framework (Phil Zampino via lmccay)
KNOX-1014 - Service Discovery and Topology Generation Framework (Phil Zampino via lmccay)
Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/c2ca4432
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/c2ca4432
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/c2ca4432
Branch: refs/heads/KNOX-1049
Commit: c2ca443262a848ae0d56e03e92ecba32fbf149f2
Parents: 3a0119b
Author: Larry McCay <lm...@hortonworks.com>
Authored: Sat Sep 23 11:04:39 2017 -0400
Committer: Larry McCay <lm...@hortonworks.com>
Committed: Sat Sep 23 11:04:39 2017 -0400
----------------------------------------------------------------------
b/gateway-discovery-ambari/pom.xml | 66 ++
.../discovery/ambari/AmbariCluster.java | 114 +++
.../discovery/ambari/AmbariComponent.java | 76 ++
.../ambari/AmbariServiceDiscovery.java | 291 +++++++
.../ambari/AmbariServiceDiscoveryMessages.java | 81 ++
.../ambari/AmbariServiceDiscoveryType.java | 35 +
.../ambari/AmbariServiceURLCreator.java | 184 ++++
...eway.topology.discovery.ServiceDiscoveryType | 19 +
.../ambari/AmbariServiceDiscoveryTest.java | 856 +++++++++++++++++++
b/gateway-release/home/conf/descriptors/README | 1 +
.../home/conf/shared-providers/README | 1 +
gateway-discovery-ambari/pom.xml | 66 ++
.../discovery/ambari/AmbariCluster.java | 114 +++
.../discovery/ambari/AmbariComponent.java | 76 ++
.../ambari/AmbariServiceDiscovery.java | 291 +++++++
.../ambari/AmbariServiceDiscoveryMessages.java | 81 ++
.../ambari/AmbariServiceDiscoveryType.java | 35 +
.../ambari/AmbariServiceURLCreator.java | 184 ++++
...eway.topology.discovery.ServiceDiscoveryType | 19 +
.../ambari/AmbariServiceDiscoveryTest.java | 856 +++++++++++++++++++
gateway-release/home/conf/descriptors/README | 1 +
.../home/conf/shared-providers/README | 1 +
gateway-release/pom.xml | 4 +
gateway-server/pom.xml | 5 +
.../apache/hadoop/gateway/GatewayMessages.java | 9 +-
.../services/DefaultGatewayServices.java | 3 +-
.../topology/impl/DefaultTopologyService.java | 278 +++++-
.../builder/BeanPropertyTopologyBuilder.java | 2 +-
.../DefaultServiceDiscoveryConfig.java | 48 ++
.../discovery/ServiceDiscoveryFactory.java | 81 ++
.../topology/simple/SimpleDescriptor.java | 46 +
.../simple/SimpleDescriptorFactory.java | 71 ++
.../simple/SimpleDescriptorHandler.java | 186 ++++
.../topology/simple/SimpleDescriptorImpl.java | 111 +++
.../simple/SimpleDescriptorMessages.java | 44 +
.../topology/DefaultTopologyServiceTest.java | 70 +-
.../PropertiesFileServiceDiscoveryTest.java | 90 ++
.../discovery/ServiceDiscoveryFactoryTest.java | 81 ++
.../test/extension/DummyServiceDiscovery.java | 66 ++
.../extension/DummyServiceDiscoveryType.java | 32 +
.../PropertiesFileServiceDiscovery.java | 108 +++
.../PropertiesFileServiceDiscoveryType.java | 35 +
.../extension/SneakyServiceDiscoveryImpl.java | 40 +
.../extension/SneakyServiceDiscoveryType.java | 33 +
.../simple/SimpleDescriptorFactoryTest.java | 218 +++++
.../simple/SimpleDescriptorHandlerTest.java | 239 ++++++
...eway.topology.discovery.ServiceDiscoveryType | 21 +
.../topology/file/ambari-cluster-policy.xml | 74 ++
.../topology/file/simple-topology-four.json | 18 +
.../topology/discovery/GatewayService.java | 29 +
.../topology/discovery/ServiceDiscovery.java | 76 ++
.../discovery/ServiceDiscoveryConfig.java | 42 +
.../discovery/ServiceDiscoveryType.java | 40 +
pom.xml | 27 +-
54 files changed, 5626 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/pom.xml
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/pom.xml b/b/gateway-discovery-ambari/pom.xml
new file mode 100644
index 0000000..924e89c
--- /dev/null
+++ b/b/gateway-discovery-ambari/pom.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.knox</groupId>
+ <artifactId>gateway</artifactId>
+ <version>0.14.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>gateway-discovery-ambari</artifactId>
+
+ <name>gateway-discovery-ambari</name>
+ <description>The extension to the gateway for service discovery using Apache Ambari.</description>
+
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+
+ <dependencies>
+ <dependency>
+ <groupId>${gateway-group}</groupId>
+ <artifactId>gateway-spi</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>${gateway-group}</groupId>
+ <artifactId>gateway-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java
new file mode 100644
index 0000000..6eaabd3
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+class AmbariCluster implements ServiceDiscovery.Cluster {
+
+ private String name = null;
+
+ private AmbariServiceURLCreator urlCreator = new AmbariServiceURLCreator();
+
+ private Map<String, Map<String, ServiceConfiguration>> serviceConfigurations = new HashMap<>();
+
+ private Map<String, AmbariComponent> components = null;
+
+
+ AmbariCluster(String name) {
+ this.name = name;
+ components = new HashMap<String, AmbariComponent>();
+ }
+
+ void addServiceConfiguration(String serviceName, String configurationType, ServiceConfiguration serviceConfig) {
+ if (!serviceConfigurations.keySet().contains(serviceName)) {
+ serviceConfigurations.put(serviceName, new HashMap<String, ServiceConfiguration>());
+ }
+ serviceConfigurations.get(serviceName).put(configurationType, serviceConfig);
+ }
+
+
+ void addComponent(AmbariComponent component) {
+ components.put(component.getName(), component);
+ }
+
+
+ ServiceConfiguration getServiceConfiguration(String serviceName, String configurationType) {
+ ServiceConfiguration sc = null;
+ Map<String, ServiceConfiguration> configs = serviceConfigurations.get(serviceName);
+ if (configs != null) {
+ sc = configs.get(configurationType);
+ }
+ return sc;
+ }
+
+
+ Map<String, AmbariComponent> getComponents() {
+ return components;
+ }
+
+
+ AmbariComponent getComponent(String name) {
+ return components.get(name);
+ }
+
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+
+ @Override
+ public List<String> getServiceURLs(String serviceName) {
+ List<String> urls = new ArrayList<>();
+ urls.addAll(urlCreator.create(this, serviceName));
+ return urls;
+ }
+
+
+ static class ServiceConfiguration {
+
+ private String type;
+ private String version;
+ private Map<String, String> props;
+
+ ServiceConfiguration(String type, String version, Map<String, String> properties) {
+ this.type = type;
+ this.version = version;
+ this.props = properties;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public Map<String, String> getProperties() {
+ return props;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariComponent.java
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariComponent.java b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariComponent.java
new file mode 100644
index 0000000..55257fb
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariComponent.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import java.util.List;
+import java.util.Map;
+
+class AmbariComponent {
+
+ private String clusterName = null;
+ private String serviceName = null;
+ private String name = null;
+ private String version = null;
+
+ private List<String> hostNames = null;
+
+ private Map<String, String> properties = null;
+
+ AmbariComponent(String name,
+ String version,
+ String cluster,
+ String service,
+ List<String> hostNames,
+ Map<String, String> properties) {
+ this.name = name;
+ this.serviceName = service;
+ this.clusterName = cluster;
+ this.version = version;
+ this.hostNames = hostNames;
+ this.properties = properties;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public List<String> getHostNames() {
+ return hostNames;
+ }
+
+ public Map<String, String> getConfigProperties() {
+ return properties;
+ }
+
+ public String getConfigProperty(String propertyName) {
+ return properties.get(propertyName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java
new file mode 100644
index 0000000..34f20a7
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java
@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import net.minidev.json.JSONArray;
+import net.minidev.json.JSONObject;
+import net.minidev.json.JSONValue;
+import org.apache.hadoop.gateway.config.ConfigurationException;
+import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
+import org.apache.hadoop.gateway.services.security.AliasService;
+import org.apache.hadoop.gateway.services.security.AliasServiceException;
+import org.apache.hadoop.gateway.topology.discovery.GatewayService;
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery;
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryConfig;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.util.EntityUtils;
+
+import java.io.IOException;
+import java.util.*;
+
+
+class AmbariServiceDiscovery implements ServiceDiscovery {
+
+ static final String TYPE = "AMBARI";
+
+ static final String AMBARI_CLUSTERS_URI = "/api/v1/clusters";
+
+ static final String AMBARI_HOSTROLES_URI =
+ AMBARI_CLUSTERS_URI + "/%s/services?fields=components/host_components/HostRoles";
+
+ static final String AMBARI_SERVICECONFIGS_URI =
+ AMBARI_CLUSTERS_URI + "/%s/configurations/service_config_versions?is_current=true";
+
+ // Map of component names to service configuration types
+ private static Map<String, String> componentServiceConfigs = new HashMap<>();
+ static {
+ componentServiceConfigs.put("NAMENODE", "hdfs-site");
+ componentServiceConfigs.put("RESOURCEMANAGER", "yarn-site");
+ componentServiceConfigs.put("OOZIE_SERVER", "oozie-site");
+ componentServiceConfigs.put("HIVE_SERVER", "hive-site");
+ componentServiceConfigs.put("WEBHCAT_SERVER", "webhcat-site");
+ componentServiceConfigs.put("HBASE_MASTER", "hbase-site");
+ } // TODO: Are there other service components, for which the endpoints can be discovered via Ambari?
+
+ private static final String DEFAULT_USER_ALIAS = "ambari.discovery.user";
+ private static final String DEFAULT_PWD_ALIAS = "ambari.discovery.password";
+
+ private static AmbariServiceURLCreator urlCreator = new AmbariServiceURLCreator();
+
+ private AmbariServiceDiscoveryMessages log = MessagesFactory.get(AmbariServiceDiscoveryMessages.class);
+
+ @GatewayService
+ private AliasService aliasService;
+
+ private CloseableHttpClient httpClient = null;
+
+ private Map<String, Map<String, String>> serviceConfiguration = new HashMap<>();
+
+
+ AmbariServiceDiscovery() {
+ httpClient = org.apache.http.impl.client.HttpClients.createDefault();
+ }
+
+
+ @Override
+ public String getType() {
+ return TYPE;
+ }
+
+
+ @Override
+ public Map<String, Cluster> discover(ServiceDiscoveryConfig config) {
+ Map<String, Cluster> clusters = new HashMap<String, Cluster>();
+
+ String discoveryAddress = config.getAddress();
+
+ // Invoke Ambari REST API to discover the available clusters
+ String clustersDiscoveryURL = String.format("%s" + AMBARI_CLUSTERS_URI, discoveryAddress);
+
+ JSONObject json = invokeREST(clustersDiscoveryURL, config.getUser(), config.getPasswordAlias());
+
+ // Parse the cluster names from the response, and perform the cluster discovery
+ JSONArray clusterItems = (JSONArray) json.get("items");
+ for (Object clusterItem : clusterItems) {
+ String clusterName = (String) ((JSONObject)((JSONObject) clusterItem).get("Clusters")).get("cluster_name");
+ try {
+ Cluster c = discover(config, clusterName);
+ clusters.put(clusterName, c);
+ } catch (Exception e) {
+ log.clusterDiscoveryError(clusterName, e);
+ }
+ }
+
+ return clusters;
+ }
+
+
+ @Override
+ public Cluster discover(ServiceDiscoveryConfig config, String clusterName) {
+ AmbariCluster cluster = new AmbariCluster(clusterName);
+
+ Map<String, String> serviceComponents = new HashMap<>();
+
+ String discoveryAddress = config.getAddress();
+ String discoveryUser = config.getUser();
+ String discoveryPwdAlias = config.getPasswordAlias();
+
+ Map<String, List<String>> componentHostNames = new HashMap<>();
+ String hostRolesURL = String.format("%s" + AMBARI_HOSTROLES_URI, discoveryAddress, clusterName);
+ JSONObject hostRolesJSON = invokeREST(hostRolesURL, discoveryUser, discoveryPwdAlias);
+ if (hostRolesJSON != null) {
+ // Process the host roles JSON
+ JSONArray items = (JSONArray) hostRolesJSON.get("items");
+ for (Object obj : items) {
+ JSONArray components = (JSONArray) ((JSONObject) obj).get("components");
+ for (Object component : components) {
+ JSONArray hostComponents = (JSONArray) ((JSONObject) component).get("host_components");
+ for (Object hostComponent : hostComponents) {
+ JSONObject hostRoles = (JSONObject) ((JSONObject) hostComponent).get("HostRoles");
+ String serviceName = (String) hostRoles.get("service_name");
+ String componentName = (String) hostRoles.get("component_name");
+
+ serviceComponents.put(componentName, serviceName);
+
+// String hostName = (String) hostRoles.get("host_name");
+ String hostName = (String) hostRoles.get("public_host_name"); // Assuming public host name is most applicable
+ log.discoveredServiceHost(serviceName, hostName);
+ if (!componentHostNames.containsKey(componentName)) {
+ componentHostNames.put(componentName, new ArrayList<String>());
+ }
+ componentHostNames.get(componentName).add(hostName);
+ }
+ }
+ }
+ }
+
+ Map<String, Map<String, AmbariCluster.ServiceConfiguration>> serviceConfigurations =
+ new HashMap<String, Map<String, AmbariCluster.ServiceConfiguration>>();
+ String serviceConfigsURL = String.format("%s" + AMBARI_SERVICECONFIGS_URI, discoveryAddress, clusterName);
+ JSONObject serviceConfigsJSON = invokeREST(serviceConfigsURL, discoveryUser, discoveryPwdAlias);
+ if (serviceConfigsJSON != null) {
+ // Process the service configurations
+ JSONArray serviceConfigs = (JSONArray) serviceConfigsJSON.get("items");
+ for (Object serviceConfig : serviceConfigs) {
+ String serviceName = (String) ((JSONObject) serviceConfig).get("service_name");
+ JSONArray configurations = (JSONArray) ((JSONObject) serviceConfig).get("configurations");
+ for (Object configuration : configurations) {
+ String configType = (String) ((JSONObject) configuration).get("type");
+ String configVersion = String.valueOf(((JSONObject) configuration).get("version"));
+
+ Map<String, String> configProps = new HashMap<String, String>();
+ JSONObject configProperties = (JSONObject) ((JSONObject) configuration).get("properties");
+ for (String propertyName : configProperties.keySet()) {
+ configProps.put(propertyName, String.valueOf(((JSONObject) configProperties).get(propertyName)));
+ }
+ if (!serviceConfigurations.containsKey(serviceName)) {
+ serviceConfigurations.put(serviceName, new HashMap<String, AmbariCluster.ServiceConfiguration>());
+ }
+ serviceConfigurations.get(serviceName).put(configType, new AmbariCluster.ServiceConfiguration(configType, configVersion, configProps));
+ cluster.addServiceConfiguration(serviceName, configType, new AmbariCluster.ServiceConfiguration(configType, configVersion, configProps));
+ }
+ }
+ }
+
+ // Construct the AmbariCluster model
+ for (String componentName : serviceComponents.keySet()) {
+ String serviceName = serviceComponents.get(componentName);
+ List<String> hostNames = componentHostNames.get(componentName);
+
+ Map<String, AmbariCluster.ServiceConfiguration> configs = serviceConfigurations.get(serviceName);
+ String configType = componentServiceConfigs.get(componentName);
+ if (configType != null) {
+ AmbariCluster.ServiceConfiguration svcConfig = configs.get(configType);
+ AmbariComponent c = new AmbariComponent(componentName,
+ svcConfig.getVersion(),
+ clusterName,
+ serviceName,
+ hostNames,
+ svcConfig.getProperties());
+ cluster.addComponent(c);
+ }
+ }
+
+ return cluster;
+ }
+
+
+ protected JSONObject invokeREST(String url, String username, String passwordAlias) {
+ JSONObject result = null;
+
+ CloseableHttpResponse response = null;
+ try {
+ HttpGet request = new HttpGet(url);
+
+ // If no configured username, then use default username alias
+ String password = null;
+ if (username == null) {
+ if (aliasService != null) {
+ try {
+ char[] defaultUser = aliasService.getPasswordFromAliasForGateway(DEFAULT_USER_ALIAS);
+ if (defaultUser != null) {
+ username = new String(defaultUser);
+ }
+ } catch (AliasServiceException e) {
+ log.aliasServiceUserError(DEFAULT_USER_ALIAS, e.getLocalizedMessage());
+ }
+ }
+
+ // If username is still null
+ if (username == null) {
+ log.aliasServiceUserNotFound();
+ throw new ConfigurationException("No username is configured for Ambari service discovery.");
+ }
+ }
+
+ if (aliasService != null) {
+ // If not password alias is configured, then try the default alias
+ if (passwordAlias == null) {
+ passwordAlias = DEFAULT_PWD_ALIAS;
+ }
+ try {
+ char[] pwd = aliasService.getPasswordFromAliasForGateway(passwordAlias);
+ if (pwd != null) {
+ password = new String(pwd);
+ }
+
+ } catch (AliasServiceException e) {
+ log.aliasServicePasswordError(passwordAlias, e.getLocalizedMessage());
+ }
+ }
+
+ // If the password could not be determined
+ if (password == null) {
+ log.aliasServicePasswordNotFound();
+ throw new ConfigurationException("No password is configured for Ambari service discovery.");
+ }
+
+ // Add an auth header if credentials are available
+ String encodedCreds =
+ org.apache.commons.codec.binary.Base64.encodeBase64String((username + ":" + password).getBytes());
+ request.addHeader(new BasicHeader("Authorization", "Basic " + encodedCreds));
+
+ response = httpClient.execute(request);
+
+ if (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) {
+ HttpEntity entity = response.getEntity();
+ if (entity != null) {
+ result = (JSONObject) JSONValue.parse((EntityUtils.toString(entity)));
+ log.debugJSON(result.toJSONString());
+ } else {
+ log.noJSON(url);
+ }
+ } else {
+ log.unexpectedRestResponseStatusCode(url, response.getStatusLine().getStatusCode());
+ }
+
+ } catch (IOException e) {
+ log.restInvocationError(url, e);
+ } finally {
+ if(response != null) {
+ try {
+ response.close();
+ } catch (IOException e) {
+ // Ignore
+ }
+ }
+ }
+ return result;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java
new file mode 100644
index 0000000..caa16ed
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import org.apache.hadoop.gateway.i18n.messages.Message;
+import org.apache.hadoop.gateway.i18n.messages.MessageLevel;
+import org.apache.hadoop.gateway.i18n.messages.Messages;
+import org.apache.hadoop.gateway.i18n.messages.StackTrace;
+
+@Messages(logger="org.apache.gateway.topology.discovery.ambari")
+public interface AmbariServiceDiscoveryMessages {
+
+ @Message(level = MessageLevel.ERROR,
+ text = "Encountered an error during cluster {0} discovery: {1}")
+ void clusterDiscoveryError(final String clusterName,
+ @StackTrace(level = MessageLevel.ERROR) Exception e);
+
+
+ @Message(level = MessageLevel.DEBUG,
+ text = "REST invocation {0} failed: {1}")
+ void restInvocationError(final String url,
+ @StackTrace(level = MessageLevel.ERROR) Exception e);
+
+
+ @Message(level = MessageLevel.ERROR,
+ text = "Encountered an error attempting to determine the user for alias {0} : {1}")
+ void aliasServiceUserError(final String alias, final String error);
+
+
+ @Message(level = MessageLevel.ERROR,
+ text = "Encountered an error attempting to determine the password for alias {0} : {1}")
+ void aliasServicePasswordError(final String alias, final String error);
+
+
+ @Message(level = MessageLevel.ERROR,
+ text = "No user configured for Ambari service discovery.")
+ void aliasServiceUserNotFound();
+
+
+ @Message(level = MessageLevel.ERROR,
+ text = "No password configured for Ambari service discovery.")
+ void aliasServicePasswordNotFound();
+
+
+ @Message(level = MessageLevel.ERROR,
+ text = "Unexpected REST invocation response code for {0} : {1}")
+ void unexpectedRestResponseStatusCode(final String url, int responseStatusCode);
+
+
+ @Message(level = MessageLevel.ERROR,
+ text = "REST invocation {0} yielded a response without any JSON.")
+ void noJSON(final String url);
+
+
+ @Message(level = MessageLevel.DEBUG,
+ text = "REST invocation result: {0}")
+ void debugJSON(final String json);
+
+
+ @Message(level = MessageLevel.INFO,
+ text = "Discovered: Service: {0}, Host: {1}")
+ void discoveredServiceHost(final String serviceName, final String hostName);
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java
new file mode 100644
index 0000000..723a786
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery;
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType;
+
+public class AmbariServiceDiscoveryType implements ServiceDiscoveryType {
+
+ private static final String IMPL = AmbariServiceDiscovery.class.getCanonicalName();
+
+ @Override
+ public String getType() {
+ return AmbariServiceDiscovery.TYPE;
+ }
+
+ @Override
+ public ServiceDiscovery newInstance() {
+ return new AmbariServiceDiscovery();
+ }
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceURLCreator.java
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceURLCreator.java b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceURLCreator.java
new file mode 100644
index 0000000..0674642
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceURLCreator.java
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+class AmbariServiceURLCreator {
+
+ private static final String NAMENODE_SERVICE = "NAMENODE";
+ private static final String JOBTRACKER_SERVICE = "JOBTRACKER";
+ private static final String WEBHDFS_SERVICE = "WEBHDFS";
+ private static final String WEBHCAT_SERVICE = "WEBHCAT";
+ private static final String OOZIE_SERVICE = "OOZIE";
+ private static final String WEBHBASE_SERVICE = "WEBHBASE";
+ private static final String HIVE_SERVICE = "HIVE";
+ private static final String RESOURCEMANAGER_SERVICE = "RESOURCEMANAGER";
+
+
+ /**
+ * Derive the endpoint URL(s) for the specified service, based on the info from the specified Cluster.
+ *
+ * @param cluster The cluster discovery results
+ * @param serviceName The name of a Hadoop service
+ *
+ * @return One or more endpoint URLs for the specified service.
+ */
+ public List<String> create(AmbariCluster cluster, String serviceName) {
+ List<String> result = null;
+
+ if (NAMENODE_SERVICE.equals(serviceName)) {
+ result = createNameNodeURL(cluster);
+ } else if (JOBTRACKER_SERVICE.equals(serviceName)) {
+ result = createJobTrackerURL(cluster);
+ } else if (WEBHDFS_SERVICE.equals(serviceName)) {
+ result = createWebHDFSURL(cluster);
+ } else if (WEBHCAT_SERVICE.equals(serviceName)) {
+ result = createWebHCatURL(cluster);
+ } else if (OOZIE_SERVICE.equals(serviceName)) {
+ result = createOozieURL(cluster);
+ } else if (WEBHBASE_SERVICE.equals(serviceName)) {
+ result = createWebHBaseURL(cluster);
+ } else if (HIVE_SERVICE.equals(serviceName)) {
+ result = createHiveURL(cluster);
+ } else if (RESOURCEMANAGER_SERVICE.equals(serviceName)) {
+ result = createResourceManagerURL(cluster);
+ }
+
+ return result;
+ }
+
+
+ private List<String> createNameNodeURL(AmbariCluster cluster) {
+ List<String> result = new ArrayList<>();
+
+ AmbariComponent comp = cluster.getComponent("NAMENODE");
+ if (comp != null) {
+ result.add("hdfs://" + comp.getConfigProperty("dfs.namenode.rpc-address"));
+ }
+
+ return result;
+ }
+
+
+ private List<String> createJobTrackerURL(AmbariCluster cluster) {
+ List<String> result = new ArrayList<>();
+
+ AmbariComponent comp = cluster.getComponent("RESOURCEMANAGER");
+ if (comp != null) {
+ result.add("rpc://" + comp.getConfigProperty("yarn.resourcemanager.address"));
+ }
+
+ return result;
+ }
+
+
+ private List<String> createWebHDFSURL(AmbariCluster cluster) {
+ List<String> result = new ArrayList<>();
+
+ AmbariCluster.ServiceConfiguration sc = cluster.getServiceConfiguration("HDFS", "hdfs-site");
+ if (sc != null) {
+ String address = sc.getProperties().get("dfs.namenode.http-address");
+ result.add("http://" + address + "/webhdfs");
+ }
+
+ return result;
+ }
+
+
+ private List<String> createWebHCatURL(AmbariCluster cluster) {
+ List<String> result = new ArrayList<>();
+
+ AmbariComponent webhcat = cluster.getComponent("WEBHCAT_SERVER");
+ if (webhcat != null) {
+ String port = webhcat.getConfigProperty("templeton.port");
+ String host = webhcat.getHostNames().get(0);
+
+ result.add("http://" + host + ":" + port + "/templeton");
+ }
+ return result;
+ }
+
+
+ private List<String> createOozieURL(AmbariCluster cluster) {
+ List<String> result = new ArrayList<>();
+
+ AmbariComponent comp = cluster.getComponent("OOZIE_SERVER");
+ if (comp != null) {
+ result.add(comp.getConfigProperty("oozie.base.url"));
+ }
+
+ return result;
+ }
+
+
+ private List<String> createWebHBaseURL(AmbariCluster cluster) {
+ List<String> result = new ArrayList<>();
+
+ AmbariComponent comp = cluster.getComponent("HBASE_MASTER");
+ if (comp != null) {
+ for (String host : comp.getHostNames()) {
+ result.add("http://" + host + ":60080");
+ }
+ }
+
+ return result;
+ }
+
+
+ private List<String> createHiveURL(AmbariCluster cluster) {
+ List<String> result = new ArrayList<>();
+
+ AmbariComponent hive = cluster.getComponent("HIVE_SERVER");
+ if (hive != null) {
+ String path = hive.getConfigProperty("hive.server2.thrift.http.path");
+ String port = hive.getConfigProperty("hive.server2.thrift.http.port");
+ String transport = hive.getConfigProperty("hive.server2.transport.mode");
+ String useSSL = hive.getConfigProperty("hive.server2.use.SSL");
+ String host = hive.getHostNames().get(0);
+
+ String scheme = null; // What is the scheme for the binary transport mode?
+ if ("http".equals(transport)) {
+ scheme = Boolean.valueOf(useSSL) ? "https" : "http";
+ }
+
+ result.add(scheme + "://" + host + ":" + port + "/" + path);
+ }
+ return result;
+ }
+
+
+ private List<String> createResourceManagerURL(AmbariCluster cluster) {
+ List<String> result = new ArrayList<>();
+
+ AmbariComponent resMan = cluster.getComponent("RESOURCEMANAGER");
+ if (resMan != null) {
+ String webappAddress = resMan.getConfigProperty("yarn.resourcemanager.webapp.address");
+ String httpPolicy = resMan.getConfigProperty("yarn.http.policy");
+ String scheme = ("HTTPS_ONLY".equalsIgnoreCase(httpPolicy)) ? "https" : "http";
+
+ result.add(scheme + "://" + webappAddress + "/ws");
+ }
+
+ return result;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType
----------------------------------------------------------------------
diff --git a/b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType b/b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType
new file mode 100644
index 0000000..1da4fc9
--- /dev/null
+++ b/b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType
@@ -0,0 +1,19 @@
+##########################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##########################################################################
+
+org.apache.hadoop.gateway.topology.discovery.ambari.AmbariServiceDiscoveryType
\ No newline at end of file