You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by lm...@apache.org on 2017/10/26 14:22:45 UTC
[09/37] knox git commit: KNOX-1014 - Service Discovery and Topology
Generation Framework (Phil Zampino via lmccay)
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java
----------------------------------------------------------------------
diff --git a/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java b/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java
new file mode 100644
index 0000000..1e5e7b2
--- /dev/null
+++ b/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java
@@ -0,0 +1,856 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery.ambari;
+
+import net.minidev.json.JSONObject;
+import net.minidev.json.JSONValue;
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery;
+import org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryConfig;
+import org.easymock.EasyMock;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+
+/**
+ * Test the Ambari ServiceDiscovery implementation.
+ *
+ * N.B. These tests do NOT verify Ambari API responses. They DO validate the Ambari ServiceDiscovery implementation's
+ * treatment of the responses as they were observed at the time the tests are developed.
+ */
+public class AmbariServiceDiscoveryTest {
+
+ @Test
+ public void testSingleClusterDiscovery() throws Exception {
+ final String discoveryAddress = "http://ambarihost:8080";
+ final String clusterName = "testCluster";
+ ServiceDiscovery sd = new TestAmbariServiceDiscovery(clusterName);
+
+ ServiceDiscoveryConfig sdc = EasyMock.createNiceMock(ServiceDiscoveryConfig.class);
+ EasyMock.expect(sdc.getAddress()).andReturn(discoveryAddress).anyTimes();
+ EasyMock.expect(sdc.getUser()).andReturn(null).anyTimes();
+ EasyMock.replay(sdc);
+
+ ServiceDiscovery.Cluster cluster = sd.discover(sdc, clusterName);
+ assertNotNull(cluster);
+ assertEquals(clusterName, cluster.getName());
+ assertTrue(AmbariCluster.class.isAssignableFrom(cluster.getClass()));
+ assertEquals(6, ((AmbariCluster) cluster).getComponents().size());
+
+// printServiceURLs(cluster);
+ }
+
+
+ @Test
+ public void testBulkClusterDiscovery() throws Exception {
+ final String discoveryAddress = "http://ambarihost:8080";
+ final String clusterName = "anotherCluster";
+ ServiceDiscovery sd = new TestAmbariServiceDiscovery(clusterName);
+
+ ServiceDiscoveryConfig sdc = EasyMock.createNiceMock(ServiceDiscoveryConfig.class);
+ EasyMock.expect(sdc.getAddress()).andReturn(discoveryAddress).anyTimes();
+ EasyMock.expect(sdc.getUser()).andReturn(null).anyTimes();
+ EasyMock.replay(sdc);
+
+ Map<String, ServiceDiscovery.Cluster> clusters = sd.discover(sdc);
+ assertNotNull(clusters);
+ assertEquals(1, clusters.size());
+ ServiceDiscovery.Cluster cluster = clusters.get(clusterName);
+ assertNotNull(cluster);
+ assertEquals(clusterName, cluster.getName());
+ assertTrue(AmbariCluster.class.isAssignableFrom(cluster.getClass()));
+ assertEquals(6, ((AmbariCluster) cluster).getComponents().size());
+
+// printServiceURLs(cluster, "NAMENODE", "WEBHCAT", "OOZIE", "RESOURCEMANAGER");
+ }
+
+
+ private static void printServiceURLs(ServiceDiscovery.Cluster cluster) {
+ final String[] services = new String[]{"NAMENODE",
+ "JOBTRACKER",
+ "WEBHDFS",
+ "WEBHCAT",
+ "OOZIE",
+ "WEBHBASE",
+ "HIVE",
+ "RESOURCEMANAGER"};
+ printServiceURLs(cluster, services);
+ }
+
+
+ private static void printServiceURLs(ServiceDiscovery.Cluster cluster, String...services) {
+ for (String name : services) {
+ StringBuilder sb = new StringBuilder();
+ List<String> urls = cluster.getServiceURLs(name);
+ if (urls != null && !urls.isEmpty()) {
+ for (String url : urls) {
+ sb.append(url);
+ sb.append(" ");
+ }
+ }
+ System.out.println(String.format("%18s: %s", name, sb.toString()));
+ }
+ }
+
+
+ /**
+ * ServiceDiscovery implementation derived from AmbariServiceDiscovery, so the invokeREST method can be overridden
+ * to eliminate the need to perform actual HTTP interactions with a real Ambari endpoint.
+ */
+ private static final class TestAmbariServiceDiscovery extends AmbariServiceDiscovery {
+
+ final static String CLUSTER_PLACEHOLDER = "CLUSTER_NAME";
+
+ private Map<String, JSONObject> cannedResponses = new HashMap<>();
+
+ TestAmbariServiceDiscovery(String clusterName) {
+ cannedResponses.put(AmbariServiceDiscovery.AMBARI_CLUSTERS_URI,
+ (JSONObject) JSONValue.parse(CLUSTERS_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER,
+ clusterName)));
+
+ cannedResponses.put(String.format(AmbariServiceDiscovery.AMBARI_HOSTROLES_URI, clusterName),
+ (JSONObject) JSONValue.parse(HOSTROLES_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER,
+ clusterName)));
+
+ cannedResponses.put(String.format(AmbariServiceDiscovery.AMBARI_SERVICECONFIGS_URI, clusterName),
+ (JSONObject) JSONValue.parse(SERVICECONFIGS_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER,
+ clusterName)));
+ }
+
+ @Override
+ protected JSONObject invokeREST(String url, String username, String passwordAlias) {
+ return cannedResponses.get(url.substring(url.indexOf("/api")));
+ }
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////
+ // JSON response templates, based on actual response content excerpts
+ ////////////////////////////////////////////////////////////////////////
+
+ private static final String CLUSTERS_JSON_TEMPLATE =
+ "{\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters\",\n" +
+ " \"items\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"Clusters\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"version\" : \"HDP-2.6\"\n" +
+ " }\n" +
+ " }\n" +
+ " ]" +
+ "}";
+
+
+ private static final String HOSTROLES_JSON_TEMPLATE =
+ "{\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services?fields=components/host_components/HostRoles\",\n" +
+ " \"items\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/AMBARI_METRICS\",\n" +
+ " \"ServiceInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"service_name\" : \"AMBARI_METRICS\"\n" +
+ " },\n" +
+ " \"components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/AMBARI_METRICS/components/METRICS_COLLECTOR\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"METRICS_COLLECTOR\",\n" +
+ " \"service_name\" : \"AMBARI_METRICS\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/METRICS_COLLECTOR\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"METRICS_COLLECTOR\",\n" +
+ " \"host_name\" : \"c6403.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6403.ambari.apache.org\",\n" +
+ " \"service_name\" : \"AMBARI_METRICS\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HBASE/components/HBASE_MASTER\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"HBASE_MASTER\",\n" +
+ " \"service_name\" : \"HBASE\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6401.ambari.apache.org/host_components/HBASE_MASTER\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"HBASE_MASTER\",\n" +
+ " \"host_name\" : \"c6401.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6401.ambari.apache.org\",\n" +
+ " \"service_name\" : \"HBASE\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HDFS\",\n" +
+ " \"ServiceInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"service_name\" : \"HDFS\"\n" +
+ " },\n" +
+ " \"components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HDFS/components/NAMENODE\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"NAMENODE\",\n" +
+ " \"service_name\" : \"HDFS\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6401.ambari.apache.org/host_components/NAMENODE\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"NAMENODE\",\n" +
+ " \"host_name\" : \"c6401.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6401.ambari.apache.org\",\n" +
+ " \"service_name\" : \"HDFS\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HDFS/components/SECONDARY_NAMENODE\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"SECONDARY_NAMENODE\",\n" +
+ " \"service_name\" : \"HDFS\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/SECONDARY_NAMENODE\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"SECONDARY_NAMENODE\",\n" +
+ " \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"service_name\" : \"HDFS\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE\",\n" +
+ " \"ServiceInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"service_name\" : \"HIVE\"\n" +
+ " },\n" +
+ " \"components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/HCAT\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"HCAT\",\n" +
+ " \"service_name\" : \"HIVE\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/HCAT\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"HCAT\",\n" +
+ " \"host_name\" : \"c6403.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6403.ambari.apache.org\",\n" +
+ " \"service_name\" : \"HIVE\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/HIVE_METASTORE\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"HIVE_METASTORE\",\n" +
+ " \"service_name\" : \"HIVE\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/HIVE_METASTORE\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"HIVE_METASTORE\",\n" +
+ " \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"service_name\" : \"HIVE\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/HIVE_SERVER\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"HIVE_SERVER\",\n" +
+ " \"service_name\" : \"HIVE\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/HIVE_SERVER\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"HIVE_SERVER\",\n" +
+ " \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"service_name\" : \"HIVE\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/WEBHCAT_SERVER\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"WEBHCAT_SERVER\",\n" +
+ " \"service_name\" : \"HIVE\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/WEBHCAT_SERVER\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"WEBHCAT_SERVER\",\n" +
+ " \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"service_name\" : \"HIVE\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/OOZIE\",\n" +
+ " \"ServiceInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"service_name\" : \"OOZIE\"\n" +
+ " },\n" +
+ " \"components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/OOZIE/components/OOZIE_SERVER\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"OOZIE_SERVER\",\n" +
+ " \"service_name\" : \"OOZIE\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/OOZIE_SERVER\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"OOZIE_SERVER\",\n" +
+ " \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"service_name\" : \"OOZIE\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN\",\n" +
+ " \"ServiceInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"service_name\" : \"YARN\"\n" +
+ " },\n" +
+ " \"components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN/components/APP_TIMELINE_SERVER\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"APP_TIMELINE_SERVER\",\n" +
+ " \"service_name\" : \"YARN\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/APP_TIMELINE_SERVER\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"APP_TIMELINE_SERVER\",\n" +
+ " \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"service_name\" : \"YARN\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN/components/NODEMANAGER\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"NODEMANAGER\",\n" +
+ " \"service_name\" : \"YARN\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/NODEMANAGER\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"NODEMANAGER\",\n" +
+ " \"host_name\" : \"c6403.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6403.ambari.apache.org\",\n" +
+ " \"service_name\" : \"YARN\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN/components/RESOURCEMANAGER\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"RESOURCEMANAGER\",\n" +
+ " \"service_name\" : \"YARN\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/RESOURCEMANAGER\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"RESOURCEMANAGER\",\n" +
+ " \"ha_state\" : \"ACTIVE\",\n" +
+ " \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"service_name\" : \"YARN\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/ZOOKEEPER\",\n" +
+ " \"ServiceInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"service_name\" : \"ZOOKEEPER\"\n" +
+ " },\n" +
+ " \"components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/ZOOKEEPER/components/ZOOKEEPER_SERVER\",\n" +
+ " \"ServiceComponentInfo\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"ZOOKEEPER_SERVER\",\n" +
+ " \"service_name\" : \"ZOOKEEPER\"\n" +
+ " },\n" +
+ " \"host_components\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6401.ambari.apache.org/host_components/ZOOKEEPER_SERVER\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"ZOOKEEPER_SERVER\",\n" +
+ " \"host_name\" : \"c6401.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6401.ambari.apache.org\",\n" +
+ " \"service_name\" : \"ZOOKEEPER\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " }\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/ZOOKEEPER_SERVER\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"ZOOKEEPER_SERVER\",\n" +
+ " \"host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" +
+ " \"service_name\" : \"ZOOKEEPER\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " }\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/ZOOKEEPER_SERVER\",\n" +
+ " \"HostRoles\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"component_name\" : \"ZOOKEEPER_SERVER\",\n" +
+ " \"host_name\" : \"c6403.ambari.apache.org\",\n" +
+ " \"public_host_name\" : \"c6403.ambari.apache.org\",\n" +
+ " \"service_name\" : \"ZOOKEEPER\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " ]\n" +
+ "}\n";
+
+
+ private static final String SERVICECONFIGS_JSON_TEMPLATE =
+ "{\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?is_current=true\",\n" +
+ " \"items\" : [\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=HBASE&service_config_version=1\",\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"configurations\" : [\n" +
+ " {\n" +
+ " \"Config\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " },\n" +
+ " \"type\" : \"hbase-site\",\n" +
+ " \"tag\" : \"version1503410563715\",\n" +
+ " \"version\" : 1,\n" +
+ " \"properties\" : {\n" +
+ " \"hbase.master.info.bindAddress\" : \"0.0.0.0\",\n" +
+ " \"hbase.master.info.port\" : \"16010\",\n" +
+ " \"hbase.master.port\" : \"16000\",\n" +
+ " \"hbase.regionserver.info.port\" : \"16030\",\n" +
+ " \"hbase.regionserver.port\" : \"16020\",\n" +
+ " \"hbase.zookeeper.property.clientPort\" : \"2181\",\n" +
+ " \"hbase.zookeeper.quorum\" : \"c6403.ambari.apache.org,c6402.ambari.apache.org,c6401.ambari.apache.org\",\n" +
+ " \"hbase.zookeeper.useMulti\" : \"true\",\n" +
+ " \"zookeeper.znode.parent\" : \"/hbase-unsecure\"\n" +
+ " },\n" +
+ " \"properties_attributes\" : { }\n" +
+ " },\n" +
+ " ],\n" +
+ " \"is_current\" : true,\n" +
+ " \"service_config_version\" : 1,\n" +
+ " \"service_config_version_note\" : \"Initial configurations for HBase\",\n" +
+ " \"service_name\" : \"HBASE\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " \"user\" : \"admin\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=HDFS&service_config_version=2\",\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"configurations\" : [\n" +
+ " {\n" +
+ " \"Config\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " },\n" +
+ " \"type\" : \"hdfs-site\",\n" +
+ " \"tag\" : \"version1\",\n" +
+ " \"version\" : 1,\n" +
+ " \"properties\" : {\n" +
+ " \"dfs.cluster.administrators\" : \" hdfs\",\n" +
+ " \"dfs.datanode.address\" : \"0.0.0.0:50010\",\n" +
+ " \"dfs.datanode.http.address\" : \"0.0.0.0:50075\",\n" +
+ " \"dfs.datanode.https.address\" : \"0.0.0.0:50475\",\n" +
+ " \"dfs.datanode.ipc.address\" : \"0.0.0.0:8010\",\n" +
+ " \"dfs.http.policy\" : \"HTTP_ONLY\",\n" +
+ " \"dfs.https.port\" : \"50470\",\n" +
+ " \"dfs.journalnode.http-address\" : \"0.0.0.0:8480\",\n" +
+ " \"dfs.journalnode.https-address\" : \"0.0.0.0:8481\",\n" +
+ " \"dfs.namenode.http-address\" : \"c6401.ambari.apache.org:50070\",\n" +
+ " \"dfs.namenode.https-address\" : \"c6401.ambari.apache.org:50470\",\n" +
+ " \"dfs.namenode.rpc-address\" : \"c6401.ambari.apache.org:8020\",\n" +
+ " \"dfs.namenode.secondary.http-address\" : \"c6402.ambari.apache.org:50090\",\n" +
+ " \"dfs.webhdfs.enabled\" : \"true\"\n" +
+ " },\n" +
+ " \"properties_attributes\" : {\n" +
+ " \"final\" : {\n" +
+ " \"dfs.webhdfs.enabled\" : \"true\",\n" +
+ " \"dfs.namenode.http-address\" : \"true\",\n" +
+ " \"dfs.support.append\" : \"true\",\n" +
+ " \"dfs.namenode.name.dir\" : \"true\",\n" +
+ " \"dfs.datanode.failed.volumes.tolerated\" : \"true\",\n" +
+ " \"dfs.datanode.data.dir\" : \"true\"\n" +
+ " }\n" +
+ " }\n" +
+ " },\n" +
+ " {\n" +
+ " \"Config\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " },\n" +
+ " \"type\" : \"core-site\",\n" +
+ " \"tag\" : \"version1502131215159\",\n" +
+ " \"version\" : 2,\n" +
+ " \"properties\" : {\n" +
+ " \"hadoop.http.authentication.simple.anonymous.allowed\" : \"true\",\n" +
+ " \"net.topology.script.file.name\" : \"/etc/hadoop/conf/topology_script.py\"\n" +
+ " },\n" +
+ " \"properties_attributes\" : {\n" +
+ " \"final\" : {\n" +
+ " \"fs.defaultFS\" : \"true\"\n" +
+ " }\n" +
+ " }\n" +
+ " }\n" +
+ " ],\n" +
+ " \"is_current\" : true,\n" +
+ " \"service_config_version\" : 2,\n" +
+ " \"service_config_version_note\" : \"knox trusted proxy support\",\n" +
+ " \"service_name\" : \"HDFS\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " \"user\" : \"admin\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=HIVE&service_config_version=3\",\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"configurations\" : [\n" +
+ " {\n" +
+ " \"Config\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " },\n" +
+ " \"type\" : \"hive-env\",\n" +
+ " \"tag\" : \"version1\",\n" +
+ " \"version\" : 1,\n" +
+ " \"properties\" : {\n" +
+ " \"hive_security_authorization\" : \"None\",\n" +
+ " \"webhcat_user\" : \"hcat\"\n" +
+ " },\n" +
+ " \"properties_attributes\" : { }\n" +
+ " },\n" +
+ " {\n" +
+ " \"Config\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " },\n" +
+ " \"type\" : \"hiveserver2-site\",\n" +
+ " \"tag\" : \"version1\",\n" +
+ " \"version\" : 1,\n" +
+ " \"properties\" : {\n" +
+ " \"hive.metastore.metrics.enabled\" : \"true\",\n" +
+ " \"hive.security.authorization.enabled\" : \"false\",\n" +
+ " \"hive.service.metrics.hadoop2.component\" : \"hiveserver2\",\n" +
+ " \"hive.service.metrics.reporter\" : \"HADOOP2\"\n" +
+ " },\n" +
+ " \"properties_attributes\" : { }\n" +
+ " },\n" +
+ " {\n" +
+ " \"Config\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " },\n" +
+ " \"type\" : \"hive-interactive-site\",\n" +
+ " \"tag\" : \"version1\",\n" +
+ " \"version\" : 1,\n" +
+ " \"properties\" : {\n" +
+ " \"hive.server2.enable.doAs\" : \"false\",\n" +
+ " \"hive.server2.tez.default.queues\" : \"default\",\n" +
+ " \"hive.server2.tez.initialize.default.sessions\" : \"true\",\n" +
+ " \"hive.server2.tez.sessions.custom.queue.allowed\" : \"ignore\",\n" +
+ " \"hive.server2.tez.sessions.per.default.queue\" : \"1\",\n" +
+ " \"hive.server2.tez.sessions.restricted.configs\" : \"hive.execution.mode,hive.execution.engine\",\n" +
+ " \"hive.server2.thrift.http.port\" : \"10501\",\n" +
+ " \"hive.server2.thrift.port\" : \"10500\",\n" +
+ " \"hive.server2.webui.port\" : \"10502\",\n" +
+ " \"hive.server2.webui.use.ssl\" : \"false\",\n" +
+ " \"hive.server2.zookeeper.namespace\" : \"hiveserver2-hive2\"\n" +
+ " },\n" +
+ " \"properties_attributes\" : { }\n" +
+ " },\n" +
+ " {\n" +
+ " \"Config\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " },\n" +
+ " \"type\" : \"tez-interactive-site\",\n" +
+ " \"tag\" : \"version1\",\n" +
+ " \"version\" : 1,\n" +
+ " \"properties\" : {\n" +
+ " \"tez.am.am-rm.heartbeat.interval-ms.max\" : \"10000\",\n" +
+ " \"tez.am.client.heartbeat.poll.interval.millis\" : \"6000\",\n" +
+ " \"tez.am.client.heartbeat.timeout.secs\" : \"90\"\n" +
+ " },\n" +
+ " \"properties_attributes\" : { }\n" +
+ " },\n" +
+ " {\n" +
+ " \"Config\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " },\n" +
+ " \"type\" : \"hive-site\",\n" +
+ " \"tag\" : \"version1502130841736\",\n" +
+ " \"version\" : 2,\n" +
+ " \"properties\" : {\n" +
+ " \"hive.metastore.sasl.enabled\" : \"false\",\n" +
+ " \"hive.metastore.server.max.threads\" : \"100000\",\n" +
+ " \"hive.metastore.uris\" : \"thrift://c6402.ambari.apache.org:9083\",\n" +
+ " \"hive.server2.allow.user.substitution\" : \"true\",\n" +
+ " \"hive.server2.authentication\" : \"NONE\",\n" +
+ " \"hive.server2.authentication.spnego.keytab\" : \"HTTP/_HOST@EXAMPLE.COM\",\n" +
+ " \"hive.server2.authentication.spnego.principal\" : \"/etc/security/keytabs/spnego.service.keytab\",\n" +
+ " \"hive.server2.enable.doAs\" : \"true\",\n" +
+ " \"hive.server2.support.dynamic.service.discovery\" : \"true\",\n" +
+ " \"hive.server2.thrift.http.path\" : \"cliservice\",\n" +
+ " \"hive.server2.thrift.http.port\" : \"10001\",\n" +
+ " \"hive.server2.thrift.max.worker.threads\" : \"500\",\n" +
+ " \"hive.server2.thrift.port\" : \"10000\",\n" +
+ " \"hive.server2.thrift.sasl.qop\" : \"auth\",\n" +
+ " \"hive.server2.transport.mode\" : \"http\",\n" +
+ " \"hive.server2.use.SSL\" : \"false\",\n" +
+ " \"hive.server2.zookeeper.namespace\" : \"hiveserver2\"\n" +
+ " },\n" +
+ " \"properties_attributes\" : {\n" +
+ " \"hidden\" : {\n" +
+ " \"javax.jdo.option.ConnectionPassword\" : \"HIVE_CLIENT,WEBHCAT_SERVER,HCAT,CONFIG_DOWNLOAD\"\n" +
+ " }\n" +
+ " }\n" +
+ " },\n" +
+ " {\n" +
+ " \"Config\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " },\n" +
+ " \"type\" : \"webhcat-site\",\n" +
+ " \"tag\" : \"version1502131111746\",\n" +
+ " \"version\" : 2,\n" +
+ " \"properties\" : {\n" +
+ " \"templeton.port\" : \"50111\",\n" +
+ " \"templeton.zookeeper.hosts\" : \"c6403.ambari.apache.org:2181,c6401.ambari.apache.org:2181,c6402.ambari.apache.org:2181\",\n" +
+ " \"webhcat.proxyuser.knox.groups\" : \"users\",\n" +
+ " \"webhcat.proxyuser.knox.hosts\" : \"*\",\n" +
+ " \"webhcat.proxyuser.root.groups\" : \"*\",\n" +
+ " \"webhcat.proxyuser.root.hosts\" : \"c6401.ambari.apache.org\"\n" +
+ " },\n" +
+ " \"properties_attributes\" : { }\n" +
+ " }\n" +
+ " ],\n" +
+ " \"createtime\" : 1502131110745,\n" +
+ " \"group_id\" : -1,\n" +
+ " \"group_name\" : \"Default\",\n" +
+ " \"hosts\" : [ ],\n" +
+ " \"is_cluster_compatible\" : true,\n" +
+ " \"is_current\" : true,\n" +
+ " \"service_config_version\" : 3,\n" +
+ " \"service_config_version_note\" : \"knox trusted proxy support\",\n" +
+ " \"service_name\" : \"HIVE\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " \"user\" : \"admin\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=OOZIE&service_config_version=3\",\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"configurations\" : [\n" +
+ " {\n" +
+ " \"Config\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " },\n" +
+ " \"type\" : \"oozie-site\",\n" +
+ " \"tag\" : \"version1502131137103\",\n" +
+ " \"version\" : 3,\n" +
+ " \"properties\" : {\n" +
+ " \"oozie.base.url\" : \"http://c6402.ambari.apache.org:11000/oozie\",\n" +
+ " },\n" +
+ " \"properties_attributes\" : { }\n" +
+ " }\n" +
+ " ],\n" +
+ " \"is_current\" : true,\n" +
+ " \"service_config_version\" : 3,\n" +
+ " \"service_name\" : \"OOZIE\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " \"user\" : \"admin\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=TEZ&service_config_version=1\",\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"configurations\" : [\n" +
+ " {\n" +
+ " \"Config\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " },\n" +
+ " \"type\" : \"tez-site\",\n" +
+ " \"tag\" : \"version1\",\n" +
+ " \"version\" : 1,\n" +
+ " \"properties\" : {\n" +
+ " \"tez.use.cluster.hadoop-libs\" : \"false\"\n" +
+ " },\n" +
+ " \"properties_attributes\" : { }\n" +
+ " }\n" +
+ " ],\n" +
+ " \"createtime\" : 1502122253525,\n" +
+ " \"group_id\" : -1,\n" +
+ " \"group_name\" : \"Default\",\n" +
+ " \"hosts\" : [ ],\n" +
+ " \"is_cluster_compatible\" : true,\n" +
+ " \"is_current\" : true,\n" +
+ " \"service_config_version\" : 1,\n" +
+ " \"service_config_version_note\" : \"Initial configurations for Tez\",\n" +
+ " \"service_name\" : \"TEZ\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " \"user\" : \"admin\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=YARN&service_config_version=1\",\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"configurations\" : [\n" +
+ " {\n" +
+ " \"Config\" : {\n" +
+ " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" +
+ " \"stack_id\" : \"HDP-2.6\"\n" +
+ " },\n" +
+ " \"type\" : \"yarn-site\",\n" +
+ " \"tag\" : \"version1\",\n" +
+ " \"version\" : 1,\n" +
+ " \"properties\" : {\n" +
+ " \"hadoop.registry.rm.enabled\" : \"true\",\n" +
+ " \"hadoop.registry.zk.quorum\" : \"c6403.ambari.apache.org:2181,c6401.ambari.apache.org:2181,c6402.ambari.apache.org:2181\",\n" +
+ " \"yarn.acl.enable\" : \"false\",\n" +
+ " \"yarn.http.policy\" : \"HTTP_ONLY\",\n" +
+ " \"yarn.nodemanager.address\" : \"0.0.0.0:45454\",\n" +
+ " \"yarn.nodemanager.bind-host\" : \"0.0.0.0\",\n" +
+ " \"yarn.resourcemanager.address\" : \"c6402.ambari.apache.org:8050\",\n" +
+ " \"yarn.resourcemanager.admin.address\" : \"c6402.ambari.apache.org:8141\",\n" +
+ " \"yarn.resourcemanager.ha.enabled\" : \"false\",\n" +
+ " \"yarn.resourcemanager.hostname\" : \"c6402.ambari.apache.org\",\n" +
+ " \"yarn.resourcemanager.resource-tracker.address\" : \"c6402.ambari.apache.org:8025\",\n" +
+ " \"yarn.resourcemanager.scheduler.address\" : \"c6402.ambari.apache.org:8030\",\n" +
+ " \"yarn.resourcemanager.webapp.address\" : \"c6402.ambari.apache.org:8088\",\n" +
+ " \"yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled\" : \"false\",\n" +
+ " \"yarn.resourcemanager.webapp.https.address\" : \"c6402.ambari.apache.org:8090\",\n" +
+ " \"yarn.resourcemanager.zk-address\" : \"c6403.ambari.apache.org:2181,c6401.ambari.apache.org:2181,c6402.ambari.apache.org:2181\"\n" +
+ " },\n" +
+ " \"properties_attributes\" : { }\n" +
+ " }\n" +
+ " ],\n" +
+ " \"is_current\" : true,\n" +
+ " \"service_config_version\" : 1,\n" +
+ " \"service_name\" : \"YARN\",\n" +
+ " \"stack_id\" : \"HDP-2.6\",\n" +
+ " \"user\" : \"admin\"\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-release/home/conf/descriptors/README
----------------------------------------------------------------------
diff --git a/gateway-release/home/conf/descriptors/README b/gateway-release/home/conf/descriptors/README
new file mode 100644
index 0000000..a2e5226
--- /dev/null
+++ b/gateway-release/home/conf/descriptors/README
@@ -0,0 +1 @@
+THIS DIRECTORY IS WHERE SIMPLE TOPOLOGY DESCRIPTORS CAN BE PLACED
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-release/home/conf/shared-providers/README
----------------------------------------------------------------------
diff --git a/gateway-release/home/conf/shared-providers/README b/gateway-release/home/conf/shared-providers/README
new file mode 100644
index 0000000..44d12a3
--- /dev/null
+++ b/gateway-release/home/conf/shared-providers/README
@@ -0,0 +1 @@
+THIS DIRECTORY IS WHERE SHARED PROVIDER CONFIGURATIONS CAN BE PLACED
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-release/pom.xml
----------------------------------------------------------------------
diff --git a/gateway-release/pom.xml b/gateway-release/pom.xml
index 83824cd..cbff307 100644
--- a/gateway-release/pom.xml
+++ b/gateway-release/pom.xml
@@ -334,6 +334,10 @@
<groupId>${gateway-group}</groupId>
<artifactId>gateway-shell-samples</artifactId>
</dependency>
+ <dependency>
+ <groupId>${gateway-group}</groupId>
+ <artifactId>gateway-discovery-ambari</artifactId>
+ </dependency>
<dependency>
<groupId>junit</groupId>
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/pom.xml
----------------------------------------------------------------------
diff --git a/gateway-server/pom.xml b/gateway-server/pom.xml
index 437d22d..5ebf793 100644
--- a/gateway-server/pom.xml
+++ b/gateway-server/pom.xml
@@ -248,6 +248,11 @@
<artifactId>javax-websocket-client-impl</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ <version>2.3.0</version>
+ </dependency>
<!-- ********** ********** ********** ********** ********** ********** -->
<!-- ********** Test Dependencies ********** -->
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
index 1f94584..6f73c1e 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
@@ -511,6 +511,11 @@ public interface GatewayMessages {
" Could not find topology \"{0}\" mapped to port \"{1}\" configured in gateway-config.xml. "
+ "This invalid topology mapping will be ignored by the gateway. "
+ "Gateway restart will be required if in the future \"{0}\" topology is added.")
- void topologyPortMappingCannotFindTopology(final String topology,
- final int port);
+ void topologyPortMappingCannotFindTopology(final String topology, final int port);
+
+
+ @Message( level = MessageLevel.ERROR, text = "An error occurred while processing {0} : {1}" )
+ void simpleDescriptorHandlingError(final String simpleDesc,
+ @StackTrace( level = MessageLevel.DEBUG ) Exception e );
+
}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java
index cefada1..02ac154 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java
@@ -105,14 +105,13 @@ public class DefaultGatewayServices implements GatewayServices {
services.put( SERVER_INFO_SERVICE, sis );
DefaultTopologyService tops = new DefaultTopologyService();
+ tops.setAliasService(alias);
tops.init( config, options );
services.put( TOPOLOGY_SERVICE, tops );
DefaultServiceDefinitionRegistry sdr = new DefaultServiceDefinitionRegistry();
sdr.init( config, options );
services.put( SERVICE_DEFINITION_REGISTRY, sdr );
- tops.init( config, options );
- services.put( TOPOLOGY_SERVICE, tops );
DefaultMetricsService metricsService = new DefaultMetricsService();
metricsService.init( config, options );
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java
index 3321f3d..a493bc4 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.gateway.config.GatewayConfig;
import org.apache.hadoop.gateway.i18n.messages.MessagesFactory;
import org.apache.hadoop.gateway.service.definition.ServiceDefinition;
import org.apache.hadoop.gateway.services.ServiceLifecycleException;
+import org.apache.hadoop.gateway.services.security.AliasService;
import org.apache.hadoop.gateway.services.topology.TopologyService;
import org.apache.hadoop.gateway.topology.Topology;
import org.apache.hadoop.gateway.topology.TopologyEvent;
@@ -45,6 +46,7 @@ import org.apache.hadoop.gateway.topology.TopologyListener;
import org.apache.hadoop.gateway.topology.TopologyMonitor;
import org.apache.hadoop.gateway.topology.TopologyProvider;
import org.apache.hadoop.gateway.topology.builder.TopologyBuilder;
+import org.apache.hadoop.gateway.topology.simple.SimpleDescriptorHandler;
import org.apache.hadoop.gateway.topology.validation.TopologyValidator;
import org.apache.hadoop.gateway.topology.xml.AmbariFormatXmlTopologyRules;
import org.apache.hadoop.gateway.topology.xml.KnoxFormatXmlTopologyRules;
@@ -74,20 +76,27 @@ import static org.apache.commons.digester3.binder.DigesterLoader.newLoader;
public class DefaultTopologyService
extends FileAlterationListenerAdaptor
implements TopologyService, TopologyMonitor, TopologyProvider, FileFilter, FileAlterationListener {
+
private static Auditor auditor = AuditServiceFactory.getAuditService().getAuditor(
AuditConstants.DEFAULT_AUDITOR_NAME, AuditConstants.KNOX_SERVICE_NAME,
AuditConstants.KNOX_COMPONENT_NAME);
+
private static final List<String> SUPPORTED_TOPOLOGY_FILE_EXTENSIONS = new ArrayList<String>();
static {
SUPPORTED_TOPOLOGY_FILE_EXTENSIONS.add("xml");
SUPPORTED_TOPOLOGY_FILE_EXTENSIONS.add("conf");
}
+
private static GatewayMessages log = MessagesFactory.get(GatewayMessages.class);
private static DigesterLoader digesterLoader = newLoader(new KnoxFormatXmlTopologyRules(), new AmbariFormatXmlTopologyRules());
- private FileAlterationMonitor monitor;
- private File directory;
+ private List<FileAlterationMonitor> monitors = new ArrayList<>();
+ private File topologiesDirectory;
+ private File descriptorsDirectory;
+
private Set<TopologyListener> listeners;
private volatile Map<File, Topology> topologies;
+ private AliasService aliasService;
+
private Topology loadTopology(File file) throws IOException, SAXException, URISyntaxException, InterruptedException {
final long TIMEOUT = 250; //ms
@@ -202,29 +211,40 @@ public class DefaultTopologyService
}
private File calculateAbsoluteTopologiesDir(GatewayConfig config) {
-
- File topoDir = new File(config.getGatewayTopologyDir());
+ String normalizedTopologyDir = FilenameUtils.normalize(config.getGatewayTopologyDir());
+ File topoDir = new File(normalizedTopologyDir);
topoDir = topoDir.getAbsoluteFile();
return topoDir;
}
- private void initListener(FileAlterationMonitor monitor, File directory) {
- this.directory = directory;
- this.monitor = monitor;
+ private File calculateAbsoluteConfigDir(GatewayConfig config) {
+ File configDir = null;
+ String path = FilenameUtils.normalize(config.getGatewayConfDir());
+ if (path != null) {
+ configDir = new File(config.getGatewayConfDir());
+ } else {
+ configDir = (new File(config.getGatewayTopologyDir())).getParentFile();
+ }
+ configDir = configDir.getAbsoluteFile();
- FileAlterationObserver observer = new FileAlterationObserver(this.directory, this);
- observer.addListener(this);
- monitor.addObserver(observer);
+ return configDir;
+ }
- this.listeners = new HashSet<>();
- this.topologies = new HashMap<>(); //loadTopologies( this.directory );
+ private void initListener(FileAlterationMonitor monitor,
+ File directory,
+ FileFilter filter,
+ FileAlterationListener listener) {
+ monitors.add(monitor);
+ FileAlterationObserver observer = new FileAlterationObserver(directory, filter);
+ observer.addListener(listener);
+ monitor.addObserver(observer);
}
- private void initListener(File directory) throws IOException, SAXException {
+ private void initListener(File directory, FileFilter filter, FileAlterationListener listener) throws IOException, SAXException {
// Increasing the monitoring interval to 5 seconds as profiling has shown
// this is rather expensive in terms of generated garbage objects.
- initListener(new FileAlterationMonitor(5000L), directory);
+ initListener(new FileAlterationMonitor(5000L), directory, filter, listener);
}
private Map<File, Topology> loadTopologies(File directory) {
@@ -261,10 +281,14 @@ public class DefaultTopologyService
return map;
}
+ public void setAliasService(AliasService as) {
+ this.aliasService = as;
+ }
+
public void deployTopology(Topology t){
try {
- File temp = new File(directory.getAbsolutePath() + "/" + t.getName() + ".xml.temp");
+ File temp = new File(topologiesDirectory.getAbsolutePath() + "/" + t.getName() + ".xml.temp");
Package topologyPkg = Topology.class.getPackage();
String pkgName = topologyPkg.getName();
String bindingFile = pkgName.replace(".", "/") + "/topology_binding-xml.xml";
@@ -277,7 +301,7 @@ public class DefaultTopologyService
mr.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
mr.marshal(t, temp);
- File topology = new File(directory.getAbsolutePath() + "/" + t.getName() + ".xml");
+ File topology = new File(topologiesDirectory.getAbsolutePath() + "/" + t.getName() + ".xml");
if(!temp.renameTo(topology)) {
FileUtils.forceDelete(temp);
throw new IOException("Could not rename temp file");
@@ -317,7 +341,7 @@ public class DefaultTopologyService
try {
synchronized (this) {
Map<File, Topology> oldTopologies = topologies;
- Map<File, Topology> newTopologies = loadTopologies(directory);
+ Map<File, Topology> newTopologies = loadTopologies(topologiesDirectory);
List<TopologyEvent> events = createChangeEvents(oldTopologies, newTopologies);
topologies = newTopologies;
notifyChangeListeners(events);
@@ -329,7 +353,7 @@ public class DefaultTopologyService
}
public void deleteTopology(Topology t) {
- File topoDir = directory;
+ File topoDir = topologiesDirectory;
if(topoDir.isDirectory() && topoDir.canRead()) {
File[] results = topoDir.listFiles();
@@ -357,8 +381,8 @@ public class DefaultTopologyService
public Map<String, List<String>> getServiceTestURLs(Topology t, GatewayConfig config) {
File tFile = null;
Map<String, List<String>> urls = new HashMap<>();
- if(directory.isDirectory() && directory.canRead()) {
- for(File f : directory.listFiles()){
+ if(topologiesDirectory.isDirectory() && topologiesDirectory.canRead()) {
+ for(File f : topologiesDirectory.listFiles()){
if(FilenameUtils.removeExtension(f.getName()).equals(t.getName())){
tFile = f;
}
@@ -387,12 +411,16 @@ public class DefaultTopologyService
@Override
public void startMonitor() throws Exception {
- monitor.start();
+ for (FileAlterationMonitor monitor : monitors) {
+ monitor.start();
+ }
}
@Override
public void stopMonitor() throws Exception {
- monitor.stop();
+ for (FileAlterationMonitor monitor : monitors) {
+ monitor.stop();
+ }
}
@Override
@@ -414,6 +442,16 @@ public class DefaultTopologyService
@Override
public void onFileDelete(java.io.File file) {
+ // For full topology descriptors, we need to make sure to delete any corresponding simple descriptors to prevent
+ // unintended subsequent generation of the topology descriptor
+ for (String ext : DescriptorsMonitor.SUPPORTED_EXTENSIONS) {
+ File simpleDesc =
+ new File(descriptorsDirectory, FilenameUtils.getBaseName(file.getName()) + "." + ext);
+ if (simpleDesc.exists()) {
+ simpleDesc.delete();
+ }
+ }
+
onFileChange(file);
}
@@ -436,12 +474,200 @@ public class DefaultTopologyService
public void init(GatewayConfig config, Map<String, String> options) throws ServiceLifecycleException {
try {
- initListener(calculateAbsoluteTopologiesDir(config));
- } catch (IOException io) {
+ listeners = new HashSet<>();
+ topologies = new HashMap<>();
+
+ topologiesDirectory = calculateAbsoluteTopologiesDir(config);
+
+ File configDirectory = calculateAbsoluteConfigDir(config);
+ descriptorsDirectory = new File(configDirectory, "descriptors");
+ File sharedProvidersDirectory = new File(configDirectory, "shared-providers");
+
+ // Add support for conf/topologies
+ initListener(topologiesDirectory, this, this);
+
+ // Add support for conf/descriptors
+ DescriptorsMonitor dm = new DescriptorsMonitor(topologiesDirectory, aliasService);
+ initListener(descriptorsDirectory,
+ dm,
+ dm);
+
+ // Add support for conf/shared-providers
+ SharedProviderConfigMonitor spm = new SharedProviderConfigMonitor(dm, descriptorsDirectory);
+ initListener(sharedProvidersDirectory, spm, spm);
+
+ } catch (IOException | SAXException io) {
throw new ServiceLifecycleException(io.getMessage());
- } catch (SAXException sax) {
- throw new ServiceLifecycleException(sax.getMessage());
+ }
+ }
+
+
+ /**
+ * Change handler for simple descriptors
+ */
+ public static class DescriptorsMonitor extends FileAlterationListenerAdaptor
+ implements FileFilter {
+
+ static final List<String> SUPPORTED_EXTENSIONS = new ArrayList<String>();
+ static {
+ SUPPORTED_EXTENSIONS.add("json");
+ SUPPORTED_EXTENSIONS.add("yml");
+ }
+
+ private File topologiesDir;
+
+ private AliasService aliasService;
+
+ private Map<String, List<String>> providerConfigReferences = new HashMap<>();
+
+
+ public DescriptorsMonitor(File topologiesDir, AliasService aliasService) {
+ this.topologiesDir = topologiesDir;
+ this.aliasService = aliasService;
+ }
+
+ List<String> getReferencingDescriptors(String providerConfigPath) {
+ List<String> result = providerConfigReferences.get(providerConfigPath);
+ if (result == null) {
+ result = Collections.emptyList();
+ }
+ return result;
+ }
+
+ @Override
+ public void onFileCreate(File file) {
+ onFileChange(file);
+ }
+
+ @Override
+ public void onFileDelete(File file) {
+ // For simple descriptors, we need to make sure to delete any corresponding full topology descriptors to trigger undeployment
+ for (String ext : DefaultTopologyService.SUPPORTED_TOPOLOGY_FILE_EXTENSIONS) {
+ File topologyFile =
+ new File(topologiesDir, FilenameUtils.getBaseName(file.getName()) + "." + ext);
+ if (topologyFile.exists()) {
+ topologyFile.delete();
+ }
+ }
+
+ String normalizedFilePath = FilenameUtils.normalize(file.getAbsolutePath());
+ String reference = null;
+ for (Map.Entry<String, List<String>> entry : providerConfigReferences.entrySet()) {
+ if (entry.getValue().contains(normalizedFilePath)) {
+ reference = entry.getKey();
+ break;
+ }
+ }
+ if (reference != null) {
+ providerConfigReferences.get(reference).remove(normalizedFilePath);
+ }
+ }
+
+ @Override
+ public void onFileChange(File file) {
+ try {
+ // When a simple descriptor has been created or modified, generate the new topology descriptor
+ Map<String, File> result = SimpleDescriptorHandler.handle(file, topologiesDir, aliasService);
+
+ // Add the provider config reference relationship for handling updates to the provider config
+ String providerConfig = FilenameUtils.normalize(result.get("reference").getAbsolutePath());
+ if (!providerConfigReferences.containsKey(providerConfig)) {
+ providerConfigReferences.put(providerConfig, new ArrayList<String>());
+ }
+ List<String> refs = providerConfigReferences.get(providerConfig);
+ String descriptorName = FilenameUtils.normalize(file.getAbsolutePath());
+ if (!refs.contains(descriptorName)) {
+ // Need to check if descriptor had previously referenced another provider config, so it can be removed
+ for (List<String> descs : providerConfigReferences.values()) {
+ if (descs.contains(descriptorName)) {
+ descs.remove(descriptorName);
+ }
+ }
+
+ // Add the current reference relationship
+ refs.add(descriptorName);
+ }
+ } catch (Exception e) {
+ log.simpleDescriptorHandlingError(file.getName(), e);
+ }
+ }
+
+ @Override
+ public boolean accept(File file) {
+ boolean accept = false;
+ if (!file.isDirectory() && file.canRead()) {
+ String extension = FilenameUtils.getExtension(file.getName());
+ if (SUPPORTED_EXTENSIONS.contains(extension)) {
+ accept = true;
+ }
+ }
+ return accept;
+ }
+ }
+
+ /**
+ * Change handler for shared provider configurations
+ */
+ public static class SharedProviderConfigMonitor extends FileAlterationListenerAdaptor
+ implements FileFilter {
+
+ static final List<String> SUPPORTED_EXTENSIONS = new ArrayList<>();
+ static {
+ SUPPORTED_EXTENSIONS.add("xml");
}
+ private DescriptorsMonitor descriptorsMonitor;
+ private File descriptorsDir;
+
+
+ SharedProviderConfigMonitor(DescriptorsMonitor descMonitor, File descriptorsDir) {
+ this.descriptorsMonitor = descMonitor;
+ this.descriptorsDir = descriptorsDir;
+ }
+
+ @Override
+ public void onFileCreate(File file) {
+ onFileChange(file);
+ }
+
+ @Override
+ public void onFileDelete(File file) {
+ onFileChange(file);
+ }
+
+ @Override
+ public void onFileChange(File file) {
+ // For shared provider configuration, we need to update any simple descriptors that reference it
+ for (File descriptor : getReferencingDescriptors(file)) {
+ descriptor.setLastModified(System.currentTimeMillis());
+ }
+ }
+
+ private List<File> getReferencingDescriptors(File sharedProviderConfig) {
+ List<File> references = new ArrayList<>();
+
+ for (File descriptor : descriptorsDir.listFiles()) {
+ if (DescriptorsMonitor.SUPPORTED_EXTENSIONS.contains(FilenameUtils.getExtension(descriptor.getName()))) {
+ for (String reference : descriptorsMonitor.getReferencingDescriptors(FilenameUtils.normalize(sharedProviderConfig.getAbsolutePath()))) {
+ references.add(new File(reference));
+ }
+ }
+ }
+
+ return references;
+ }
+
+ @Override
+ public boolean accept(File file) {
+ boolean accept = false;
+ if (!file.isDirectory() && file.canRead()) {
+ String extension = FilenameUtils.getExtension(file.getName());
+ if (SUPPORTED_EXTENSIONS.contains(extension)) {
+ accept = true;
+ }
+ }
+ return accept;
+ }
}
+
}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
index b5e80d2..a30cf13 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java
@@ -77,7 +77,7 @@ public class BeanPropertyTopologyBuilder implements TopologyBuilder {
Topology topology = new Topology();
topology.setName(name);
- for (Provider provider : providers) {
+ for (Provider provider : providers) {
topology.addProvider(provider);
}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java
new file mode 100644
index 0000000..6534b5e
--- /dev/null
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery;
+
+public class DefaultServiceDiscoveryConfig implements ServiceDiscoveryConfig {
+ private String address = null;
+ private String user = null;
+ private String pwdAlias = null;
+
+ public DefaultServiceDiscoveryConfig(String address) {
+ this.address = address;
+ }
+
+ public void setUser(String username) {
+ this.user = username;
+ }
+
+ public void setPasswordAlias(String alias) {
+ this.pwdAlias = alias;
+ }
+
+ public String getAddress() {
+ return address;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public String getPasswordAlias() {
+ return pwdAlias;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/ServiceDiscoveryFactory.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/ServiceDiscoveryFactory.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/ServiceDiscoveryFactory.java
new file mode 100644
index 0000000..70d5f61
--- /dev/null
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/ServiceDiscoveryFactory.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.discovery;
+
+import org.apache.hadoop.gateway.services.Service;
+
+import java.lang.reflect.Field;
+import java.util.ServiceLoader;
+
+/**
+ * Creates instances of ServiceDiscovery implementations.
+ *
+ * This factory uses the ServiceLoader mechanism to load ServiceDiscovery implementations as extensions.
+ *
+ */
+public abstract class ServiceDiscoveryFactory {
+
+ private static final Service[] NO_GATEWAY_SERVICS = new Service[]{};
+
+
+ public static ServiceDiscovery get(String type) {
+ return get(type, NO_GATEWAY_SERVICS);
+ }
+
+
+ public static ServiceDiscovery get(String type, Service...gatewayServices) {
+ ServiceDiscovery sd = null;
+
+ // Look up the available ServiceDiscovery types
+ ServiceLoader<ServiceDiscoveryType> loader = ServiceLoader.load(ServiceDiscoveryType.class);
+ for (ServiceDiscoveryType sdt : loader) {
+ if (sdt.getType().equalsIgnoreCase(type)) {
+ try {
+ ServiceDiscovery instance = sdt.newInstance();
+ // Make sure the type reported by the instance matches the type declared by the factory
+ // (is this necessary?)
+ if (instance.getType().equalsIgnoreCase(type)) {
+ sd = instance;
+
+ // Inject any gateway services that were specified, and which are referenced in the impl
+ if (gatewayServices != null && gatewayServices.length > 0) {
+ for (Field field : sd.getClass().getDeclaredFields()) {
+ if (field.getDeclaredAnnotation(GatewayService.class) != null) {
+ for (Service s : gatewayServices) {
+ if (s != null) {
+ if (field.getType().isAssignableFrom(s.getClass())) {
+ field.setAccessible(true);
+ field.set(sd, s);
+ }
+ }
+ }
+ }
+ }
+ }
+ break;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ return sd;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptor.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptor.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptor.java
new file mode 100644
index 0000000..aa28469
--- /dev/null
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptor.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.simple;
+
+import java.util.List;
+
+public interface SimpleDescriptor {
+
+ String getName();
+
+ String getDiscoveryType();
+
+ String getDiscoveryAddress();
+
+ String getDiscoveryUser();
+
+ String getDiscoveryPasswordAlias();
+
+ String getClusterName();
+
+ String getProviderConfig();
+
+ List<Service> getServices();
+
+
+ interface Service {
+ String getName();
+
+ List<String> getURLs();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorFactory.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorFactory.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorFactory.java
new file mode 100644
index 0000000..3df6d2f
--- /dev/null
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorFactory.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.gateway.topology.simple;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.apache.commons.io.FilenameUtils;
+
+import java.io.File;
+import java.io.IOException;
+
+
+public class SimpleDescriptorFactory {
+
+ /**
+ * Create a SimpleDescriptor from the specified file.
+ *
+ * @param path The path to the file.
+ * @return A SimpleDescriptor based on the contents of the file.
+ *
+ * @throws IOException
+ */
+ public static SimpleDescriptor parse(String path) throws IOException {
+ SimpleDescriptor sd;
+
+ if (path.endsWith(".json")) {
+ sd = parseJSON(path);
+ } else if (path.endsWith(".yml")) {
+ sd = parseYAML(path);
+ } else {
+ throw new IllegalArgumentException("Unsupported simple descriptor format: " + path.substring(path.lastIndexOf('.')));
+ }
+
+ return sd;
+ }
+
+
+ static SimpleDescriptor parseJSON(String path) throws IOException {
+ final ObjectMapper mapper = new ObjectMapper();
+ SimpleDescriptorImpl sd = mapper.readValue(new File(path), SimpleDescriptorImpl.class);
+ if (sd != null) {
+ sd.setName(FilenameUtils.getBaseName(path));
+ }
+ return sd;
+ }
+
+
+ static SimpleDescriptor parseYAML(String path) throws IOException {
+ final ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+ SimpleDescriptorImpl sd = mapper.readValue(new File(path), SimpleDescriptorImpl.class);
+ if (sd != null) {
+ sd.setName(FilenameUtils.getBaseName(path));
+ }
+ return sd;
+ }
+
+}