You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2015/01/12 15:12:04 UTC
ambari git commit: AMBARI-9086. Could not download client configs
from Host detail page.(Generation file has failed with 500 error: )
(dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/trunk c14025873 -> c73b74f67
AMBARI-9086. Could not download client configs from Host detail page.(Generation file has failed with 500 error: ) (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c73b74f6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c73b74f6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c73b74f6
Branch: refs/heads/trunk
Commit: c73b74f6764a8bdaa0b2f420c387ce2ca2a98192
Parents: c140258
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Mon Jan 12 16:11:24 2015 +0200
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Mon Jan 12 16:11:24 2015 +0200
----------------------------------------------------------------------
.../internal/ClientConfigResourceProvider.java | 17 +-
.../ClientConfigResourceProviderTest.java | 220 ++++++++++++++++++-
2 files changed, 228 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c73b74f6/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index d13b53e..0fa1b7c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -40,6 +40,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.*;
+import org.apache.ambari.server.stack.StackManager;
/**
* Resource provider for client config resources.
@@ -122,8 +123,9 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
}
Configuration configs = new Configuration();
- String TMP_PATH = configs.getProperty(Configuration.SERVER_TMP_DIR_KEY);
- String pythonCmd = configs.getProperty(Configuration.AMBARI_PYTHON_WRAP_KEY);
+ Map<String, String> configMap = configs.getConfigsMap();
+ String TMP_PATH = configMap.get(Configuration.SERVER_TMP_DIR_KEY);
+ String pythonCmd = configMap.get(Configuration.AMBARI_PYTHON_WRAP_KEY);
AmbariManagementController managementController = getManagementController();
ConfigHelper configHelper = managementController.getConfigHelper();
Cluster cluster = null;
@@ -142,7 +144,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
getComponent(stackId.getStackName(), stackId.getStackVersion(), serviceName, componentName);
packageFolder = managementController.getAmbariMetaInfo().
getService(stackId.getStackName(), stackId.getStackVersion(), serviceName).getServicePackageFolder();
-
+
String commandScript = componentInfo.getCommandScript().getScript();
List<ClientConfigFileDefinition> clientConfigFiles = componentInfo.getClientConfigFiles();
@@ -151,8 +153,13 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
}
String stackRoot = managementController.getAmbariMetaInfo().getStackRoot().getAbsolutePath();
-
- String packageFolderAbsolute = stackRoot + File.separator + packageFolder;
+ String packageFolderAbsolute = null;
+ if (packageFolder.contains(StackManager.COMMON_SERVICES)){
+ packageFolderAbsolute = configs.getCommonServicesPath().replace(StackManager.COMMON_SERVICES, packageFolder);
+ } else {
+ packageFolderAbsolute = stackRoot + File.separator + packageFolder;
+ }
+
String commandScriptAbsolute = packageFolderAbsolute + File.separator + commandScript;
http://git-wip-us.apache.org/repos/asf/ambari/blob/c73b74f6/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
index 7f353a5..9221860 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
@@ -41,6 +41,9 @@ import java.io.InputStream;
import java.io.PrintWriter;
import java.lang.reflect.Method;
import java.util.*;
+import static org.apache.ambari.server.configuration.Configuration.AMBARI_PYTHON_WRAP_DEFAULT;
+import static org.apache.ambari.server.configuration.Configuration.SERVER_TMP_DIR_DEFAULT;
+import org.apache.ambari.server.stack.StackManager;
import static org.easymock.EasyMock.*;
import static org.junit.Assert.assertFalse;
@@ -151,6 +154,7 @@ public class ClientConfigResourceProviderTest {
ServiceOsSpecific serviceOsSpecific = createNiceMock(ServiceOsSpecific.class);
ConfigHelper configHelper = createNiceMock(ConfigHelper.class);
Configuration configuration = PowerMock.createStrictMockAndExpectNew(Configuration.class);
+ Map<String, String> configMap = createNiceMock(Map.class);
File mockFile = PowerMock.createNiceMock(File.class);
Runtime runtime = createMock(Runtime.class);
@@ -219,7 +223,11 @@ public class ClientConfigResourceProviderTest {
Set<ServiceComponentHostResponse> responses = new LinkedHashSet<ServiceComponentHostResponse>();
responses.add(shr1);
-
+
+ Map<String, String> returnConfigMap = new HashMap<String, String>();
+ returnConfigMap.put(Configuration.SERVER_TMP_DIR_KEY, Configuration.SERVER_TMP_DIR_DEFAULT);
+ returnConfigMap.put(Configuration.AMBARI_PYTHON_WRAP_KEY, Configuration.AMBARI_PYTHON_WRAP_DEFAULT);
+
// set expectations
expect(managementController.getConfigHelper()).andReturn(configHelper);
expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes();
@@ -228,8 +236,9 @@ public class ClientConfigResourceProviderTest {
expect(configHelper.getEffectiveConfigProperties(cluster, configTags)).andReturn(properties);
expect(clusterConfig.getType()).andReturn(Configuration.HIVE_CONFIG_TAG).anyTimes();
expect(configHelper.getEffectiveConfigAttributes(cluster, configTags)).andReturn(attributes);
- expect(configuration.getProperty(Configuration.SERVER_TMP_DIR_KEY)).andReturn(Configuration.SERVER_TMP_DIR_DEFAULT);
- expect(configuration.getProperty(Configuration.AMBARI_PYTHON_WRAP_KEY)).andReturn(Configuration.AMBARI_PYTHON_WRAP_DEFAULT);
+ expect(configMap.get(Configuration.SERVER_TMP_DIR_KEY)).andReturn(Configuration.SERVER_TMP_DIR_DEFAULT);
+ expect(configMap.get(Configuration.AMBARI_PYTHON_WRAP_KEY)).andReturn(Configuration.AMBARI_PYTHON_WRAP_DEFAULT);
+ expect(configuration.getConfigsMap()).andReturn(returnConfigMap);
expect(configuration.getExternalScriptTimeout()).andReturn(Integer.parseInt(Configuration.EXTERNAL_SCRIPT_TIMEOUT_DEFAULT));
Map<String,String> props = new HashMap<String, String>();
props.put(Configuration.HIVE_METASTORE_PASSWORD_PROPERTY, "pass");
@@ -309,7 +318,7 @@ public class ClientConfigResourceProviderTest {
// replay
replay(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo, commandScriptDefinition,
clusterConfig, host, service, serviceComponent, serviceComponentHost, serviceInfo, configHelper,
- runtime, process);
+ runtime, process, configMap);
PowerMock.replayAll();
Set<Resource> resources = provider.getResources(request, predicate);
@@ -323,6 +332,209 @@ public class ClientConfigResourceProviderTest {
}
@Test
+ public void testGetResourcesFromCommonServices() throws Exception {
+ Resource.Type type = Resource.Type.ClientConfig;
+
+ AmbariManagementController managementController = createNiceMock(AmbariManagementController.class);
+ Clusters clusters = createNiceMock(Clusters.class);
+
+ Cluster cluster = createNiceMock(Cluster.class);
+ AmbariMetaInfo ambariMetaInfo = createNiceMock(AmbariMetaInfo.class);
+ StackId stackId = createNiceMock(StackId.class);
+ ComponentInfo componentInfo = createNiceMock(ComponentInfo.class);
+ ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class);
+ CommandScriptDefinition commandScriptDefinition = createNiceMock(CommandScriptDefinition.class);
+ Config clusterConfig = createNiceMock(Config.class);
+ DesiredConfig desiredConfig = createNiceMock(DesiredConfig.class);
+ Host host = createNiceMock(Host.class);
+ Service service = createNiceMock(Service.class);
+ ServiceComponent serviceComponent = createNiceMock(ServiceComponent.class);
+ ServiceComponentHost serviceComponentHost = createNiceMock(ServiceComponentHost.class);
+ ServiceOsSpecific serviceOsSpecific = createNiceMock(ServiceOsSpecific.class);
+ ConfigHelper configHelper = createNiceMock(ConfigHelper.class);
+ Configuration configuration = PowerMock.createStrictMockAndExpectNew(Configuration.class);
+ Map<String, String> configMap = createNiceMock(Map.class);
+
+ File mockFile = PowerMock.createNiceMock(File.class);
+ Runtime runtime = createMock(Runtime.class);
+ Process process = createNiceMock(Process.class);
+
+ Map<String, DesiredConfig> desiredConfigMap = new HashMap<String, DesiredConfig>();
+ desiredConfigMap.put("hive-site", desiredConfig);
+ Map<String, Map<String, String>> allConfigTags = new HashMap<String, Map<String, String>>();
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, Map<String, String>> configTags = new HashMap<String,
+ Map<String, String>>();
+ Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String,
+ Map<String, Map<String, String>>>();
+
+ ClientConfigFileDefinition clientConfigFileDefinition = new ClientConfigFileDefinition();
+ clientConfigFileDefinition.setDictionaryName("pig-env");
+ clientConfigFileDefinition.setFileName("pig-env.sh");
+ clientConfigFileDefinition.setType("env");
+ List <ClientConfigFileDefinition> clientConfigFileDefinitionList = new LinkedList<ClientConfigFileDefinition>();
+ clientConfigFileDefinitionList.add(clientConfigFileDefinition);
+
+ ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider(
+ type,
+ PropertyHelper.getPropertyIds(type),
+ PropertyHelper.getKeyPropertyIds(type),
+ managementController);
+
+ // create the request
+ Request request = PropertyHelper.getReadRequest(ClientConfigResourceProvider.COMPONENT_CLUSTER_NAME_PROPERTY_ID, "c1",
+ ClientConfigResourceProvider.COMPONENT_COMPONENT_NAME_PROPERTY_ID,
+ ClientConfigResourceProvider.COMPONENT_SERVICE_NAME_PROPERTY_ID);
+
+ Predicate predicate = new PredicateBuilder().property(ClientConfigResourceProvider.COMPONENT_CLUSTER_NAME_PROPERTY_ID).equals("c1").
+ toPredicate();
+
+ String clusterName = "C1";
+ String serviceName = "PIG";
+ String componentName = "PIG";
+ String hostName = "Host100";
+ String desiredState = "INSTALLED";
+
+ String stackName = "S1";
+ String stackVersion = "V1";
+
+ String stackRoot="/tmp/stacks/S1/V1";
+ String packageFolder= StackManager.COMMON_SERVICES + "/PIG/package";
+ String commonServicesPath = "/var/lib/ambari-server/src/main/resources/common-services";
+
+ if (System.getProperty("os.name").contains("Windows")) {
+ stackRoot = "\\tmp\\stacks\\S1\\V1";
+ packageFolder = StackManager.COMMON_SERVICES + "\\PIG\\package";
+ }
+
+ File stackRootFile = new File(stackRoot);
+ HashMap<String, Host> hosts = new HashMap<String, Host>();
+ hosts.put(hostName, host);
+ HashMap<String, Service> services = new HashMap<String, Service>();
+ services.put(serviceName,service);
+ HashMap<String, ServiceComponent> serviceComponentMap = new HashMap<String, ServiceComponent>();
+ serviceComponentMap.put(componentName,serviceComponent);
+ HashMap<String, ServiceComponentHost> serviceComponentHosts = new HashMap<String, ServiceComponentHost>();
+ serviceComponentHosts.put(componentName, serviceComponentHost);
+ HashMap<String, ServiceOsSpecific> serviceOsSpecificHashMap = new HashMap<String, ServiceOsSpecific>();
+ serviceOsSpecificHashMap.put("key",serviceOsSpecific);
+
+ ServiceComponentHostResponse shr1 = new ServiceComponentHostResponse(clusterName, serviceName, componentName, hostName, desiredState, "", null, null, null);
+
+ Set<ServiceComponentHostResponse> responses = new LinkedHashSet<ServiceComponentHostResponse>();
+ responses.add(shr1);
+
+ Map<String, String> returnConfigMap = new HashMap<String, String>();
+ returnConfigMap.put(Configuration.SERVER_TMP_DIR_KEY, Configuration.SERVER_TMP_DIR_DEFAULT);
+ returnConfigMap.put(Configuration.AMBARI_PYTHON_WRAP_KEY, Configuration.AMBARI_PYTHON_WRAP_DEFAULT);
+
+ // set expectations
+ expect(managementController.getConfigHelper()).andReturn(configHelper);
+ expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes();
+ expect(managementController.getClusters()).andReturn(clusters).anyTimes();
+ expect(clusters.getCluster(clusterName)).andReturn(cluster).anyTimes();
+ expect(configHelper.getEffectiveConfigProperties(cluster, configTags)).andReturn(properties);
+ expect(clusterConfig.getType()).andReturn(Configuration.HIVE_CONFIG_TAG).anyTimes();
+ expect(configHelper.getEffectiveConfigAttributes(cluster, configTags)).andReturn(attributes);
+ expect(configMap.get(Configuration.SERVER_TMP_DIR_KEY)).andReturn(Configuration.SERVER_TMP_DIR_DEFAULT);
+ expect(configMap.get(Configuration.AMBARI_PYTHON_WRAP_KEY)).andReturn(Configuration.AMBARI_PYTHON_WRAP_DEFAULT);
+ expect(configuration.getConfigsMap()).andReturn(returnConfigMap);
+ expect(configuration.getCommonServicesPath()).andReturn(commonServicesPath);
+ expect(configuration.getExternalScriptTimeout()).andReturn(Integer.parseInt(Configuration.EXTERNAL_SCRIPT_TIMEOUT_DEFAULT));
+ Map<String,String> props = new HashMap<String, String>();
+ props.put(Configuration.HIVE_METASTORE_PASSWORD_PROPERTY, "pass");
+ props.put("key","value");
+ expect(clusterConfig.getProperties()).andReturn(props);
+ expect(configHelper.getEffectiveDesiredTags(cluster, hostName)).andReturn(allConfigTags);
+ expect(cluster.getClusterName()).andReturn(clusterName);
+ expect(managementController.getHostComponents((Set<ServiceComponentHostRequest>) anyObject())).andReturn(responses).anyTimes();
+ expect(cluster.getCurrentStackVersion()).andReturn(stackId);
+
+ PowerMock.mockStaticPartial(StageUtils.class, "getClusterHostInfo");
+ Map<String, Set<String>> clusterHostInfo = new HashMap<String, Set<String>>();
+ Set<String> all_hosts = new HashSet<String>(Arrays.asList("Host100","Host101","Host102"));
+ Set<String> some_hosts = new HashSet<String>(Arrays.asList("0-1","2"));
+ Set<String> ohter_hosts = new HashSet<String>(Arrays.asList("0,1"));
+ Set<String> clusterHostTypes = new HashSet<String>(Arrays.asList("nm_hosts", "hs_host",
+ "namenode_host", "rm_host", "snamenode_host", "slave_hosts", "zookeeper_hosts"));
+ for (String hostTypes: clusterHostTypes) {
+ if (hostTypes.equals("slave_hosts")) {
+ clusterHostInfo.put(hostTypes, ohter_hosts);
+ } else {
+ clusterHostInfo.put(hostTypes, some_hosts);
+ }
+ }
+ Map<String, Host> stringHostMap = new HashMap<String, Host>();
+ stringHostMap.put(hostName, host);
+ clusterHostInfo.put("all_hosts",all_hosts);
+ expect(StageUtils.getClusterHostInfo(stringHostMap,cluster)).andReturn(clusterHostInfo);
+
+ expect(stackId.getStackName()).andReturn(stackName).anyTimes();
+ expect(stackId.getStackVersion()).andReturn(stackVersion).anyTimes();
+
+ expect(ambariMetaInfo.getComponent(stackName, stackVersion, serviceName, componentName)).andReturn(componentInfo);
+ expect(ambariMetaInfo.getService(stackName, stackVersion, serviceName)).andReturn(serviceInfo);
+ expect(serviceInfo.getServicePackageFolder()).andReturn(packageFolder);
+ expect(ambariMetaInfo.getComponent((String) anyObject(), (String) anyObject(),
+ (String) anyObject(), (String) anyObject())).andReturn(componentInfo).anyTimes();
+ expect(componentInfo.getCommandScript()).andReturn(commandScriptDefinition);
+ expect(componentInfo.getClientConfigFiles()).andReturn(clientConfigFileDefinitionList);
+ expect(ambariMetaInfo.getStackRoot()).andReturn(stackRootFile);
+ expect(cluster.getConfig("hive-site", null)).andReturn(clusterConfig);
+ expect(cluster.getDesiredConfigs()).andReturn(desiredConfigMap);
+ expect(clusters.getHostsForCluster(clusterName)).andReturn(hosts);
+ expect(clusters.getHost(hostName)).andReturn(host);
+
+ HashMap<String, String> rcaParams = new HashMap<String, String>();
+ rcaParams.put("key","value");
+ expect(managementController.getRcaParameters()).andReturn(rcaParams).anyTimes();
+ expect(ambariMetaInfo.getService(stackName, stackVersion, serviceName)).andReturn(serviceInfo);
+ expect(serviceInfo.getOsSpecifics()).andReturn(new HashMap<String, ServiceOsSpecific>()).anyTimes();
+ Set<String> userSet = new HashSet<String>();
+ userSet.add("hdfs");
+ expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER, cluster)).andReturn(userSet);
+ PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(mockFile).anyTimes();
+ PowerMock.createNiceMockAndExpectNew(PrintWriter.class, anyObject());
+ expect(mockFile.getParent()).andReturn("");
+ PowerMock.mockStatic(Runtime.class);
+ expect(mockFile.exists()).andReturn(true);
+ String commandLine = "ambari-python-wrap " + commonServicesPath + "/PIG/package/null generate_configs null " +
+ commonServicesPath + "/PIG/package /var/lib/ambari-server/tmp/structured-out.json " +
+ "INFO /var/lib/ambari-server/tmp";
+
+ if (System.getProperty("os.name").contains("Windows")) {
+ String absoluteStackRoot = stackRootFile.getAbsolutePath();
+ commandLine = "ambari-python-wrap " + commonServicesPath +
+ "\\PIG\\package\\null generate_configs null " +
+ commonServicesPath + "\\PIG\\package /var/lib/ambari-server/tmp\\structured-out.json " +
+ "INFO /var/lib/ambari-server/tmp";
+ }
+
+ ProcessBuilder processBuilder = PowerMock.createNiceMock(ProcessBuilder.class);
+ PowerMock.expectNew(ProcessBuilder.class,Arrays.asList(commandLine.split("\\s+"))).andReturn(processBuilder).once();
+ expect(processBuilder.start()).andReturn(process).once();
+ InputStream inputStream = new ByteArrayInputStream("some logging info".getBytes());
+ expect(process.getInputStream()).andReturn(inputStream);
+
+ // replay
+ replay(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo, commandScriptDefinition,
+ clusterConfig, host, service, serviceComponent, serviceComponentHost, serviceInfo, configHelper,
+ runtime, process, configMap);
+ PowerMock.replayAll();
+
+ Set<Resource> resources = provider.getResources(request, predicate);
+ assertFalse(resources.isEmpty());
+
+ // verify
+ verify(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo,commandScriptDefinition,
+ clusterConfig, host, service, serviceComponent, serviceComponentHost, serviceInfo, configHelper,
+ runtime, process);
+ PowerMock.verifyAll();
+ }
+
+
+
+ @Test
public void testDeleteResources() throws Exception {
Resource.Type type = Resource.Type.ClientConfig;