You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2015/04/01 01:35:35 UTC
[2/2] ambari git commit: AMBARI-10207. BE: Extend stack-advisor to
recommend property_value_attributes (mpapyrkovskyy via srimanth)
AMBARI-10207. BE: Extend stack-advisor to recommend property_value_attributes (mpapyrkovskyy via srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d7fcd258
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d7fcd258
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d7fcd258
Branch: refs/heads/trunk
Commit: d7fcd258d9788923a03d0d121320e0d39636a2cd
Parents: b95fe44
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Tue Mar 31 16:31:37 2015 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Tue Mar 31 16:31:44 2015 -0700
----------------------------------------------------------------------
.../server/api/services/AmbariMetaInfo.java | 41 ++++
.../stackadvisor/StackAdvisorHelper.java | 20 +-
.../stackadvisor/StackAdvisorRequest.java | 21 +-
.../ComponentLayoutRecommendationCommand.java | 67 ++++++
.../ComponentLayoutValidationCommand.java | 63 ++++++
...rationDependenciesRecommendationCommand.java | 119 ++++++++++
.../ConfigurationRecommendationCommand.java | 113 ++++++++++
.../ConfigurationValidationCommand.java | 62 ++++++
...GetComponentLayoutRecommnedationCommand.java | 64 ------
.../GetComponentLayoutValidationCommand.java | 63 ------
.../GetConfigurationRecommnedationCommand.java | 113 ----------
.../GetConfigurationValidationCommand.java | 62 ------
.../commands/StackAdvisorCommand.java | 39 ++--
.../commands/StackAdvisorCommandType.java | 2 +
.../recommendations/RecommendationResponse.java | 36 ++-
.../internal/StackAdvisorResourceProvider.java | 34 ++-
.../apache/ambari/server/stack/StackModule.java | 14 +-
.../configuration-mapred/mapred-site.xml | 16 +-
.../YARN/2.1.0.2.0/configuration/yarn-site.xml | 4 +-
.../src/main/resources/properties.json | 1 +
.../src/main/resources/scripts/stack_advisor.py | 50 ++++-
.../stacks/BIGTOP/0.8/services/stack_advisor.py | 2 +-
.../stacks/HDP/2.0.6/services/stack_advisor.py | 58 ++++-
.../services/YARN/configuration/yarn-site.xml | 4 +-
.../stacks/HDP/2.2/services/stack_advisor.py | 28 ++-
.../stacks/HDPWIN/2.1/services/stack_advisor.py | 2 +-
.../PHD/3.0.0.0/services/stack_advisor.py | 2 +-
.../src/main/resources/stacks/stack_advisor.py | 4 +-
.../server/api/services/AmbariMetaInfoTest.java | 20 +-
.../stackadvisor/StackAdvisorHelperTest.java | 33 ++-
.../ConfigurationRecommendationCommandTest.java | 103 +++++++++
...tConfigurationRecommnedationCommandTest.java | 103 ---------
.../stacks/2.0.6/common/test_stack_advisor.py | 8 +-
.../stacks/2.1/common/test_stack_advisor.py | 2 +-
.../stacks/2.2/common/test_stack_advisor.py | 222 ++++++++++++++++++-
.../services/YARN/configuration/yarn-site.xml | 17 +-
36 files changed, 1123 insertions(+), 489 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
index 0cb8fa4..3a7fb6a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
@@ -59,8 +59,10 @@ import org.apache.ambari.server.stack.StackManager;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.DependencyInfo;
import org.apache.ambari.server.state.OperatingSystemInfo;
+import org.apache.ambari.server.state.PropertyDependencyInfo;
import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.Service;
@@ -1221,4 +1223,43 @@ public class AmbariMetaInfo {
return kerberosServiceDescriptors;
}
+
+ /**
+ * Get set of all the depended-by properties down to directed acyclic graph(DAG)
+ * of dependencies between configuration properties
+ * @param stackName the stack name
+ * @param stackVersion the stack version
+ * @param changedConfigs the list of changed configurations
+ * @return set of all depended-by properties including all changedConfigs
+ */
+ public Set<PropertyDependencyInfo> getDependedByProperties(String stackName,
+ String stackVersion,
+ List<PropertyDependencyInfo> changedConfigs) {
+ StackInfo stack = getStackManager().getStack(stackName, stackVersion);
+
+ if (changedConfigs == null) {
+ return Collections.emptySet();
+ }
+ int size = 0;
+ Set<PropertyDependencyInfo> configs =
+ new HashSet<PropertyDependencyInfo>();
+
+ configs.addAll(changedConfigs);
+
+ while (size != configs.size()) {
+ size = configs.size();
+ for (ServiceInfo service: stack.getServices()) {
+ for (PropertyInfo pi: service.getProperties()) {
+ String type = ConfigHelper.fileNameToConfigType(pi.getFilename());
+ String name = pi.getName();
+ PropertyDependencyInfo dep =
+ new PropertyDependencyInfo(type, name);
+ if (configs.contains(dep) && !configs.containsAll(pi.getDependedByProperties())) {
+ configs.addAll(pi.getDependedByProperties());
+ }
+ }
+ }
+ }
+ return configs;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
index 9e683f0..a925d7d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
@@ -23,10 +23,11 @@ import java.io.IOException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType;
-import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutRecommnedationCommand;
-import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutValidationCommand;
-import org.apache.ambari.server.api.services.stackadvisor.commands.GetConfigurationRecommnedationCommand;
-import org.apache.ambari.server.api.services.stackadvisor.commands.GetConfigurationValidationCommand;
+import org.apache.ambari.server.api.services.stackadvisor.commands.ComponentLayoutRecommendationCommand;
+import org.apache.ambari.server.api.services.stackadvisor.commands.ComponentLayoutValidationCommand;
+import org.apache.ambari.server.api.services.stackadvisor.commands.ConfigurationDependenciesRecommendationCommand;
+import org.apache.ambari.server.api.services.stackadvisor.commands.ConfigurationRecommendationCommand;
+import org.apache.ambari.server.api.services.stackadvisor.commands.ConfigurationValidationCommand;
import org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand;
import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
@@ -77,10 +78,10 @@ public class StackAdvisorHelper {
StackAdvisorRequestType requestType) throws StackAdvisorException {
StackAdvisorCommand<ValidationResponse> command;
if (requestType == StackAdvisorRequestType.HOST_GROUPS) {
- command = new GetComponentLayoutValidationCommand(recommendationsDir, stackAdvisorScript,
+ command = new ComponentLayoutValidationCommand(recommendationsDir, stackAdvisorScript,
requestId, saRunner, metaInfo);
} else if (requestType == StackAdvisorRequestType.CONFIGURATIONS) {
- command = new GetConfigurationValidationCommand(recommendationsDir, stackAdvisorScript,
+ command = new ConfigurationValidationCommand(recommendationsDir, stackAdvisorScript,
requestId, saRunner, metaInfo);
} else {
throw new StackAdvisorRequestException(String.format("Unsupported request type, type=%s",
@@ -112,10 +113,13 @@ public class StackAdvisorHelper {
StackAdvisorRequestType requestType) throws StackAdvisorException {
StackAdvisorCommand<RecommendationResponse> command;
if (requestType == StackAdvisorRequestType.HOST_GROUPS) {
- command = new GetComponentLayoutRecommnedationCommand(recommendationsDir, stackAdvisorScript,
+ command = new ComponentLayoutRecommendationCommand(recommendationsDir, stackAdvisorScript,
requestId, saRunner, metaInfo);
} else if (requestType == StackAdvisorRequestType.CONFIGURATIONS) {
- command = new GetConfigurationRecommnedationCommand(recommendationsDir, stackAdvisorScript,
+ command = new ConfigurationRecommendationCommand(recommendationsDir, stackAdvisorScript,
+ requestId, saRunner, metaInfo);
+ } else if (requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES) {
+ command = new ConfigurationDependenciesRecommendationCommand(recommendationsDir, stackAdvisorScript,
requestId, saRunner, metaInfo);
} else {
throw new StackAdvisorRequestException(String.format("Unsupported request type, type=%s",
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
index 991f198..89a3570 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
@@ -21,10 +21,12 @@ package org.apache.ambari.server.api.services.stackadvisor;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.ambari.server.state.PropertyDependencyInfo;
import org.apache.commons.lang.StringUtils;
/**
@@ -41,6 +43,7 @@ public class StackAdvisorRequest {
private Map<String, Set<String>> hostComponents = new HashMap<String, Set<String>>();
private Map<String, Set<String>> hostGroupBindings = new HashMap<String, Set<String>>();
private Map<String, Map<String, Map<String, String>>> configurations = new HashMap<String, Map<String, Map<String, String>>>();
+ private List<PropertyDependencyInfo> changedConfigurations = new LinkedList<PropertyDependencyInfo>();
public String getStackName() {
return stackName;
@@ -86,6 +89,14 @@ public class StackAdvisorRequest {
return configurations;
}
+ public List<PropertyDependencyInfo> getChangedConfigurations() {
+ return changedConfigurations;
+ }
+
+ public void setChangedConfigurations(List<PropertyDependencyInfo> changedConfigurations) {
+ this.changedConfigurations = changedConfigurations;
+ }
+
private StackAdvisorRequest(String stackName, String stackVersion) {
this.stackName = stackName;
this.stackVersion = stackVersion;
@@ -140,13 +151,21 @@ public class StackAdvisorRequest {
return this;
}
+ public StackAdvisorRequestBuilder withChangedConfigurations(
+ List<PropertyDependencyInfo> changedConfigurations) {
+ this.instance.changedConfigurations = changedConfigurations;
+ return this;
+ }
+
public StackAdvisorRequest build() {
return this.instance;
}
}
public enum StackAdvisorRequestType {
- HOST_GROUPS("host_groups"), CONFIGURATIONS("configurations");
+ HOST_GROUPS("host_groups"),
+ CONFIGURATIONS("configurations"),
+ CONFIGURATION_DEPENDENCIES("configuration-dependencies");
private String type;
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutRecommendationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutRecommendationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutRecommendationCommand.java
new file mode 100644
index 0000000..0dff92b
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutRecommendationCommand.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.api.services.stackadvisor.commands;
+
+import java.io.File;
+
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
+import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
+
+/**
+ * {@link StackAdvisorCommand} implementation for component-layout
+ * recommendation.
+ */
+public class ComponentLayoutRecommendationCommand extends
+ StackAdvisorCommand<RecommendationResponse> {
+
+ public ComponentLayoutRecommendationCommand(File recommendationsDir,
+ String stackAdvisorScript,
+ int requestId,
+ StackAdvisorRunner saRunner,
+ AmbariMetaInfo metaInfo) {
+ super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo);
+ }
+
+ @Override
+ protected StackAdvisorCommandType getCommandType() {
+ return StackAdvisorCommandType.RECOMMEND_COMPONENT_LAYOUT;
+ }
+
+ @Override
+ protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
+ if (request.getHosts() == null || request.getHosts().isEmpty()
+ || request.getServices() == null || request.getServices().isEmpty()) {
+ throw new StackAdvisorException("Hosts and services must not be empty");
+ }
+ }
+
+ @Override
+ protected RecommendationResponse updateResponse(StackAdvisorRequest request, RecommendationResponse response) {
+ return response;
+ }
+
+ @Override
+ protected String getResultFileName() {
+ return "component-layout.json";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutValidationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutValidationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutValidationCommand.java
new file mode 100644
index 0000000..757ebee
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ComponentLayoutValidationCommand.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.api.services.stackadvisor.commands;
+
+import java.io.File;
+
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
+import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
+
+/**
+ * {@link StackAdvisorCommand} implementation for component-layout validation.
+ */
+public class ComponentLayoutValidationCommand extends StackAdvisorCommand<ValidationResponse> {
+
+ public ComponentLayoutValidationCommand(File recommendationsDir, String stackAdvisorScript,
+ int requestId, StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) {
+ super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo);
+ }
+
+ @Override
+ protected StackAdvisorCommandType getCommandType() {
+ return StackAdvisorCommandType.VALIDATE_COMPONENT_LAYOUT;
+ }
+
+ @Override
+ protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
+ if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null
+ || request.getServices().isEmpty() || request.getComponentHostsMap() == null
+ || request.getComponentHostsMap().isEmpty()) {
+ throw new StackAdvisorException("Hosts, services and recommendations must not be empty");
+ }
+ }
+
+ @Override
+ protected ValidationResponse updateResponse(StackAdvisorRequest request, ValidationResponse response) {
+ return response;
+ }
+
+ @Override
+ protected String getResultFileName() {
+ return "component-layout-validation.json";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationDependenciesRecommendationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationDependenciesRecommendationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationDependenciesRecommendationCommand.java
new file mode 100644
index 0000000..ae86548
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationDependenciesRecommendationCommand.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.api.services.stackadvisor.commands;
+
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
+import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.BindingHostGroup;
+import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.HostGroup;
+
+/**
+ * {@link StackAdvisorCommand} implementation for
+ * configuration dependencies recommendation.
+ */
+public class ConfigurationDependenciesRecommendationCommand extends
+ StackAdvisorCommand<RecommendationResponse> {
+
+ public ConfigurationDependenciesRecommendationCommand(File recommendationsDir, String stackAdvisorScript, int requestId,
+ StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) {
+ super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo);
+ }
+
+ @Override
+ protected StackAdvisorCommandType getCommandType() {
+ return StackAdvisorCommandType.RECOMMEND_CONFIGURATION_DEPENDENCIES;
+ }
+
+ @Override
+ protected void validate(StackAdvisorRequest request)
+ throws StackAdvisorException {
+ if (request.getHosts() == null || request.getHosts().isEmpty()
+ || request.getServices() == null || request.getServices().isEmpty()
+ || request.getChangedConfigurations() == null
+ || request.getChangedConfigurations().isEmpty()) {
+ throw new StackAdvisorException("Hosts, services and changed-configurations must not be empty");
+ }
+ }
+
+ @Override
+ protected RecommendationResponse updateResponse(StackAdvisorRequest request,
+ RecommendationResponse response) {
+ response.getRecommendations().getBlueprint().setHostGroups(processHostGroups(request));
+ response.getRecommendations().getBlueprintClusterBinding().setHostGroups(processHostGroupBindings(request));
+ return response;
+ }
+
+ protected Set<HostGroup> processHostGroups(StackAdvisorRequest request) {
+ Set<HostGroup> resultSet = new HashSet<HostGroup>();
+ for (Map.Entry<String, Set<String>> componentHost : request.getHostComponents().entrySet()) {
+ String hostGroupName = componentHost.getKey();
+ Set<String> components = componentHost.getValue();
+ if (hostGroupName != null && components != null) {
+ HostGroup hostGroup = new HostGroup();
+ Set<Map<String, String>> componentsSet = new HashSet<Map<String, String>>();
+ for (String component : components) {
+ Map<String, String> componentMap = new HashMap<String, String>();
+ componentMap.put("name", component);
+ componentsSet.add(componentMap);
+ }
+ hostGroup.setComponents(componentsSet);
+ hostGroup.setName(hostGroupName);
+ resultSet.add(hostGroup);
+ }
+ }
+ return resultSet;
+ }
+
+ private Set<BindingHostGroup> processHostGroupBindings(StackAdvisorRequest request) {
+ Set<BindingHostGroup> resultSet = new HashSet<BindingHostGroup>();
+ for (Map.Entry<String, Set<String>> hostBinding : request.getHostGroupBindings().entrySet()) {
+ String hostGroupName = hostBinding.getKey();
+ Set<String> hosts = hostBinding.getValue();
+ if (hostGroupName != null && hosts != null) {
+ BindingHostGroup bindingHostGroup = new BindingHostGroup();
+ Set<Map<String, String>> hostsSet = new HashSet<Map<String, String>>();
+ for (String host : hosts) {
+ Map<String, String> hostMap = new HashMap<String, String>();
+ hostMap.put("name", host);
+ hostsSet.add(hostMap);
+ }
+ bindingHostGroup.setHosts(hostsSet);
+ bindingHostGroup.setName(hostGroupName);
+ resultSet.add(bindingHostGroup);
+ }
+ }
+ return resultSet;
+ }
+
+ @Override
+ protected String getResultFileName() {
+ return "configurations.json";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationRecommendationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationRecommendationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationRecommendationCommand.java
new file mode 100644
index 0000000..ad01b40
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationRecommendationCommand.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.api.services.stackadvisor.commands;
+
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
+import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
+import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.*;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * {@link org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand} implementation for
+ * configuration recommendation.
+ */
+public class ConfigurationRecommendationCommand extends
+ StackAdvisorCommand<RecommendationResponse> {
+
+ public ConfigurationRecommendationCommand(File recommendationsDir, String stackAdvisorScript, int requestId,
+ StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) {
+ super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo);
+ }
+
+ @Override
+ protected StackAdvisorCommandType getCommandType() {
+ return StackAdvisorCommandType.RECOMMEND_CONFIGURATIONS;
+ }
+
+ @Override
+ protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
+ if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null
+ || request.getServices().isEmpty()) {
+ throw new StackAdvisorException("Hosts and services must not be empty");
+ }
+ }
+
+ @Override
+ protected RecommendationResponse updateResponse(StackAdvisorRequest request, RecommendationResponse response) {
+ response.getRecommendations().getBlueprint().setHostGroups(processHostGroups(request));
+ response.getRecommendations().getBlueprintClusterBinding().setHostGroups(processHostGroupBindings(request));
+ return response;
+ }
+
+ protected Set<HostGroup> processHostGroups(StackAdvisorRequest request) {
+ Set<HostGroup> resultSet = new HashSet<HostGroup>();
+ for (Map.Entry<String, Set<String>> componentHost : request.getHostComponents().entrySet()) {
+ String hostGroupName = componentHost.getKey();
+ Set<String> components = componentHost.getValue();
+ if (hostGroupName != null && components != null) {
+ HostGroup hostGroup = new HostGroup();
+ Set<Map<String, String>> componentsSet = new HashSet<Map<String, String>>();
+ for (String component : components) {
+ Map<String, String> componentMap = new HashMap<String, String>();
+ componentMap.put("name", component);
+ componentsSet.add(componentMap);
+ }
+ hostGroup.setComponents(componentsSet);
+ hostGroup.setName(hostGroupName);
+ resultSet.add(hostGroup);
+ }
+ }
+ return resultSet;
+ }
+
+ private Set<BindingHostGroup> processHostGroupBindings(StackAdvisorRequest request) {
+ Set<BindingHostGroup> resultSet = new HashSet<BindingHostGroup>();
+ for (Map.Entry<String, Set<String>> hostBinding : request.getHostGroupBindings().entrySet()) {
+ String hostGroupName = hostBinding.getKey();
+ Set<String> hosts = hostBinding.getValue();
+ if (hostGroupName != null && hosts != null) {
+ BindingHostGroup bindingHostGroup = new BindingHostGroup();
+ Set<Map<String, String>> hostsSet = new HashSet<Map<String, String>>();
+ for (String host : hosts) {
+ Map<String, String> hostMap = new HashMap<String, String>();
+ hostMap.put("name", host);
+ hostsSet.add(hostMap);
+ }
+ bindingHostGroup.setHosts(hostsSet);
+ bindingHostGroup.setName(hostGroupName);
+ resultSet.add(bindingHostGroup);
+ }
+ }
+ return resultSet;
+ }
+
+ @Override
+ protected String getResultFileName() {
+ return "configurations.json";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationValidationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationValidationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationValidationCommand.java
new file mode 100644
index 0000000..c234947
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/ConfigurationValidationCommand.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.api.services.stackadvisor.commands;
+
+import java.io.File;
+
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
+import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
+
+/**
+ * {@link StackAdvisorCommand} implementation for configuration validation.
+ */
+public class ConfigurationValidationCommand extends StackAdvisorCommand<ValidationResponse> {
+
+ public ConfigurationValidationCommand(File recommendationsDir, String stackAdvisorScript,
+ int requestId, StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) {
+ super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo);
+ }
+
+ @Override
+ protected StackAdvisorCommandType getCommandType() {
+ return StackAdvisorCommandType.VALIDATE_CONFIGURATIONS;
+ }
+
+ @Override
+ protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
+ if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null
+ || request.getServices().isEmpty()) {
+ throw new StackAdvisorException("Hosts, services and configurations must not be empty");
+ }
+ }
+
+ @Override
+ protected ValidationResponse updateResponse(StackAdvisorRequest request,
+ ValidationResponse response) {
+ return response;
+ }
+
+ @Override
+ protected String getResultFileName() {
+ return "configurations-validation.json";
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java
deleted file mode 100644
index b91f912..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.api.services.stackadvisor.commands;
-
-import java.io.File;
-
-import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
-import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
-import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
-import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
-
-/**
- * {@link StackAdvisorCommand} implementation for component-layout
- * recommendation.
- */
-public class GetComponentLayoutRecommnedationCommand extends
- StackAdvisorCommand<RecommendationResponse> {
-
- public GetComponentLayoutRecommnedationCommand(File recommendationsDir,
- String stackAdvisorScript, int requestId, StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) {
- super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo);
- }
-
- @Override
- protected StackAdvisorCommandType getCommandType() {
- return StackAdvisorCommandType.RECOMMEND_COMPONENT_LAYOUT;
- }
-
- @Override
- protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
- if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null
- || request.getServices().isEmpty()) {
- throw new StackAdvisorException("Hosts and services must not be empty");
- }
- }
-
- @Override
- protected RecommendationResponse updateResponse(StackAdvisorRequest request, RecommendationResponse response) {
- return response;
- }
-
- @Override
- protected String getResultFileName() {
- return "component-layout.json";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java
deleted file mode 100644
index 1a1fc98..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.api.services.stackadvisor.commands;
-
-import java.io.File;
-
-import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
-import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
-import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
-import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
-
-/**
- * {@link StackAdvisorCommand} implementation for component-layout validation.
- */
-public class GetComponentLayoutValidationCommand extends StackAdvisorCommand<ValidationResponse> {
-
- public GetComponentLayoutValidationCommand(File recommendationsDir, String stackAdvisorScript,
- int requestId, StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) {
- super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo);
- }
-
- @Override
- protected StackAdvisorCommandType getCommandType() {
- return StackAdvisorCommandType.VALIDATE_COMPONENT_LAYOUT;
- }
-
- @Override
- protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
- if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null
- || request.getServices().isEmpty() || request.getComponentHostsMap() == null
- || request.getComponentHostsMap().isEmpty()) {
- throw new StackAdvisorException("Hosts, services and recommendations must not be empty");
- }
- }
-
- @Override
- protected ValidationResponse updateResponse(StackAdvisorRequest request, ValidationResponse response) {
- return response;
- }
-
- @Override
- protected String getResultFileName() {
- return "component-layout-validation.json";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
deleted file mode 100644
index b20c966..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.api.services.stackadvisor.commands;
-
-import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
-import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
-import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
-import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
-import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.*;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * {@link org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand} implementation for
- * configuration recommendation.
- */
-public class GetConfigurationRecommnedationCommand extends
- StackAdvisorCommand<RecommendationResponse> {
-
- public GetConfigurationRecommnedationCommand(File recommendationsDir, String stackAdvisorScript, int requestId,
- StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) {
- super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo);
- }
-
- @Override
- protected StackAdvisorCommandType getCommandType() {
- return StackAdvisorCommandType.RECOMMEND_CONFIGURATIONS;
- }
-
- @Override
- protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
- if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null
- || request.getServices().isEmpty()) {
- throw new StackAdvisorException("Hosts and services must not be empty");
- }
- }
-
- @Override
- protected RecommendationResponse updateResponse(StackAdvisorRequest request, RecommendationResponse response) {
- response.getRecommendations().getBlueprint().setHostGroups(processHostGroups(request));
- response.getRecommendations().getBlueprintClusterBinding().setHostGroups(processHostGroupBindings(request));
- return response;
- }
-
- protected Set<HostGroup> processHostGroups(StackAdvisorRequest request) {
- Set<HostGroup> resultSet = new HashSet<HostGroup>();
- for (Map.Entry<String, Set<String>> componentHost : request.getHostComponents().entrySet()) {
- String hostGroupName = componentHost.getKey();
- Set<String> components = componentHost.getValue();
- if (hostGroupName != null && components != null) {
- HostGroup hostGroup = new HostGroup();
- Set<Map<String, String>> componentsSet = new HashSet<Map<String, String>>();
- for (String component : components) {
- Map<String, String> componentMap = new HashMap<String, String>();
- componentMap.put("name", component);
- componentsSet.add(componentMap);
- }
- hostGroup.setComponents(componentsSet);
- hostGroup.setName(hostGroupName);
- resultSet.add(hostGroup);
- }
- }
- return resultSet;
- }
-
- private Set<BindingHostGroup> processHostGroupBindings(StackAdvisorRequest request) {
- Set<BindingHostGroup> resultSet = new HashSet<BindingHostGroup>();
- for (Map.Entry<String, Set<String>> hostBinding : request.getHostGroupBindings().entrySet()) {
- String hostGroupName = hostBinding.getKey();
- Set<String> hosts = hostBinding.getValue();
- if (hostGroupName != null && hosts != null) {
- BindingHostGroup bindingHostGroup = new BindingHostGroup();
- Set<Map<String, String>> hostsSet = new HashSet<Map<String, String>>();
- for (String host : hosts) {
- Map<String, String> hostMap = new HashMap<String, String>();
- hostMap.put("name", host);
- hostsSet.add(hostMap);
- }
- bindingHostGroup.setHosts(hostsSet);
- bindingHostGroup.setName(hostGroupName);
- resultSet.add(bindingHostGroup);
- }
- }
- return resultSet;
- }
-
- @Override
- protected String getResultFileName() {
- return "configurations.json";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java
deleted file mode 100644
index 36fe6cb..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.api.services.stackadvisor.commands;
-
-import java.io.File;
-
-import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
-import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
-import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
-import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
-
-/**
- * {@link StackAdvisorCommand} implementation for configuration validation.
- */
-public class GetConfigurationValidationCommand extends StackAdvisorCommand<ValidationResponse> {
-
- public GetConfigurationValidationCommand(File recommendationsDir, String stackAdvisorScript,
- int requestId, StackAdvisorRunner saRunner, AmbariMetaInfo metaInfo) {
- super(recommendationsDir, stackAdvisorScript, requestId, saRunner, metaInfo);
- }
-
- @Override
- protected StackAdvisorCommandType getCommandType() {
- return StackAdvisorCommandType.VALIDATE_CONFIGURATIONS;
- }
-
- @Override
- protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
- if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices() == null
- || request.getServices().isEmpty()) {
- throw new StackAdvisorException("Hosts, services and configurations must not be empty");
- }
- }
-
- @Override
- protected ValidationResponse updateResponse(StackAdvisorRequest request,
- ValidationResponse response) {
- return response;
- }
-
- @Override
- protected String getResultFileName() {
- return "configurations-validation.json";
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
index 2e20452..a7646a0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
@@ -44,6 +44,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorResponse;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.state.PropertyDependencyInfo;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
@@ -75,12 +76,14 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
+ ",services/StackServices/service_name,services/StackServices/service_version"
+ ",services/components/StackServiceComponents,services/components/dependencies,services/components/auto_deploy"
+ "&services/StackServices/service_name.in(%s)";
- private static final String SERVICES_PROPETRY = "services";
- private static final String SERVICES_COMPONENTS_PROPETRY = "components";
- private static final String COMPONENT_INFO_PROPETRY = "StackServiceComponents";
+ private static final String SERVICES_PROPERTY = "services";
+ private static final String SERVICES_COMPONENTS_PROPERTY = "components";
+ private static final String COMPONENT_INFO_PROPERTY = "StackServiceComponents";
private static final String COMPONENT_NAME_PROPERTY = "component_name";
- private static final String COMPONENT_HOSTNAMES_PROPETRY = "hostnames";
- private static final String CONFIGURATIONS_PROPETRY = "configurations";
+ private static final String COMPONENT_HOSTNAMES_PROPERTY = "hostnames";
+ private static final String CONFIGURATIONS_PROPERTY = "configurations";
+ private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations";
+ private static final String DEPENDED_CONFIGURATIONS_PROPERTY = "depended-configurations";
private File recommendationsDir;
private String stackAdvisorScript;
@@ -140,8 +143,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
populateStackHierarchy(root);
populateComponentHostsMap(root, request.getComponentHostsMap());
- populateConfigurations(root, request.getConfigurations());
-
+ populateConfigurations(root, request);
data.servicesJSON = mapper.writeValueAsString(root);
} catch (Exception e) {
// should not happen
@@ -154,8 +156,10 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
}
private void populateConfigurations(ObjectNode root,
- Map<String, Map<String, Map<String, String>>> configurations) {
- ObjectNode configurationsNode = root.putObject(CONFIGURATIONS_PROPETRY);
+ StackAdvisorRequest request) {
+ Map<String, Map<String, Map<String, String>>> configurations =
+ request.getConfigurations();
+ ObjectNode configurationsNode = root.putObject(CONFIGURATIONS_PROPERTY);
for (String siteName : configurations.keySet()) {
ObjectNode siteNode = configurationsNode.putObject(siteName);
@@ -170,6 +174,15 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
}
}
}
+ // Populating changed configs info and depended by configs info
+ Set<PropertyDependencyInfo> dependedProperties =
+ metaInfo.getDependedByProperties(request.getStackName(),
+ request.getStackVersion(), request.getChangedConfigurations());
+
+ JsonNode changedConfigs = mapper.valueToTree(request.getChangedConfigurations());
+ JsonNode dependendConfigs = mapper.valueToTree(dependedProperties);
+ root.put(CHANGED_CONFIGURATIONS_PROPERTY, changedConfigs);
+ root.put(DEPENDED_CONFIGURATIONS_PROPERTY, dependendConfigs);
}
protected void populateStackHierarchy(ObjectNode root) {
@@ -185,21 +198,21 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
}
private void populateComponentHostsMap(ObjectNode root, Map<String, Set<String>> componentHostsMap) {
- ArrayNode services = (ArrayNode) root.get(SERVICES_PROPETRY);
+ ArrayNode services = (ArrayNode) root.get(SERVICES_PROPERTY);
Iterator<JsonNode> servicesIter = services.getElements();
while (servicesIter.hasNext()) {
JsonNode service = servicesIter.next();
- ArrayNode components = (ArrayNode) service.get(SERVICES_COMPONENTS_PROPETRY);
+ ArrayNode components = (ArrayNode) service.get(SERVICES_COMPONENTS_PROPERTY);
Iterator<JsonNode> componentsIter = components.getElements();
while (componentsIter.hasNext()) {
JsonNode component = componentsIter.next();
- ObjectNode componentInfo = (ObjectNode) component.get(COMPONENT_INFO_PROPETRY);
+ ObjectNode componentInfo = (ObjectNode) component.get(COMPONENT_INFO_PROPERTY);
String componentName = componentInfo.get(COMPONENT_NAME_PROPERTY).getTextValue();
Set<String> componentHosts = componentHostsMap.get(componentName);
- ArrayNode hostnames = componentInfo.putArray(COMPONENT_HOSTNAMES_PROPETRY);
+ ArrayNode hostnames = componentInfo.putArray(COMPONENT_HOSTNAMES_PROPERTY);
if (null != componentHosts) {
for (String hostName : componentHosts) {
hostnames.add(hostName);
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandType.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandType.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandType.java
index 380b81d..5aa1ea9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandType.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandType.java
@@ -29,6 +29,8 @@ public enum StackAdvisorCommandType {
RECOMMEND_CONFIGURATIONS("recommend-configurations"),
+ RECOMMEND_CONFIGURATION_DEPENDENCIES("recommend-configuration-dependencies"),
+
VALIDATE_CONFIGURATIONS("validate-configurations");
private final String name;
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java
index 1796f9a..b81320d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java
@@ -23,6 +23,7 @@ import java.util.Set;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorResponse;
import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
/**
* Recommendation response POJO.
@@ -88,16 +89,16 @@ public class RecommendationResponse extends StackAdvisorResponse {
public static class Blueprint {
@JsonProperty
- private Map<String, Map<String, Map<String, String>>> configurations;
+ private Map<String, BlueprintConfigurations> configurations;
@JsonProperty("host_groups")
private Set<HostGroup> hostGroups;
- public Map<String, Map<String, Map<String, String>>> getConfigurations() {
+ public Map<String, BlueprintConfigurations> getConfigurations() {
return configurations;
}
- public void setConfigurations(Map<String, Map<String, Map<String, String>>> configurations) {
+ public void setConfigurations(Map<String, BlueprintConfigurations> configurations) {
this.configurations = configurations;
}
@@ -110,6 +111,35 @@ public class RecommendationResponse extends StackAdvisorResponse {
}
}
+ public static class BlueprintConfigurations {
+ @JsonProperty
+ private Map<String, String> properties;
+
+ @JsonProperty("property_attributes")
+ @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+ private Map<String, Map<String, String>> propertyAttributes;
+
+ public BlueprintConfigurations() {
+ System.out.println(this);
+ }
+
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ public Map<String, Map<String, String>> getPropertyAttributes() {
+ return propertyAttributes;
+ }
+
+ public void setPropertyAttributes(Map<String, Map<String, String>> propertyAttributes) {
+ this.propertyAttributes = propertyAttributes;
+ }
+ }
+
public static class HostGroup {
@JsonProperty
private String name;
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
index 6ae557d..620251b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
@@ -18,8 +18,10 @@
package org.apache.ambari.server.controller.internal;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -38,6 +40,7 @@ import org.apache.ambari.server.controller.spi.Resource.Type;
import org.apache.ambari.server.controller.utilities.PropertyHelper;
import com.google.inject.Inject;
+import org.apache.ambari.server.state.PropertyDependencyInfo;
/**
* Abstract superclass for recommendations and validations.
@@ -52,6 +55,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
private static final String HOST_PROPERTY = "hosts";
private static final String SERVICES_PROPERTY = "services";
+ private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed_configurations";
+
private static final String BLUEPRINT_HOST_GROUPS_PROPERTY = "recommendations/blueprint/host_groups";
private static final String BINDING_HOST_GROUPS_PROPERTY = "recommendations/blueprint_cluster_binding/host_groups";
@@ -99,12 +104,17 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
hgHostsMap);
Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request);
- StackAdvisorRequest saRequest = StackAdvisorRequestBuilder.forStack(stackName, stackVersion)
- .ofType(requestType).forHosts(hosts).forServices(services)
- .forHostComponents(hgComponentsMap).forHostsGroupBindings(hgHostsMap)
- .withComponentHostsMap(componentHostsMap).withConfigurations(configurations).build();
-
- return saRequest;
+ List<PropertyDependencyInfo> changedConfigurations =
+ requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES ?
+ calculateChangedConfigurations(request) : Collections.<PropertyDependencyInfo>emptyList();
+
+ return StackAdvisorRequestBuilder.
+ forStack(stackName, stackVersion).ofType(requestType).forHosts(hosts).
+ forServices(services).forHostComponents(hgComponentsMap).
+ forHostsGroupBindings(hgHostsMap).
+ withComponentHostsMap(componentHostsMap).
+ withConfigurations(configurations).
+ withChangedConfigurations(changedConfigurations).build();
} catch (Exception e) {
LOG.warn("Error occured during preparation of stack advisor request", e);
Response response = Response.status(Status.BAD_REQUEST)
@@ -176,6 +186,18 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
return map;
}
+ protected List<PropertyDependencyInfo> calculateChangedConfigurations(Request request) {
+ List<PropertyDependencyInfo> configs =
+ new LinkedList<PropertyDependencyInfo>();
+ HashSet<HashMap<String, String>> changedConfigs =
+ (HashSet<HashMap<String, String>>) getRequestProperty(request, CHANGED_CONFIGURATIONS_PROPERTY);
+ for (HashMap<String, String> props: changedConfigs) {
+ configs.add(new PropertyDependencyInfo(props.get("type"), props.get("name")));
+ }
+
+ return configs;
+ }
+
protected static final String CONFIGURATIONS_PROPERTY_ID = "recommendations/blueprint/configurations/";
protected Map<String, Map<String, Map<String, String>>> calculateConfigurations(Request request) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
index 0b27ae5..4b88aff 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
@@ -29,6 +29,7 @@ import java.util.Set;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.PropertyDependencyInfo;
import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.RepositoryInfo;
@@ -597,11 +598,15 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
for (ServiceModule serviceModule : serviceModules.values()) {
for (PropertyInfo pi : serviceModule.getModuleInfo().getProperties()) {
for (PropertyDependencyInfo pdi : pi.getDependsOnProperties()) {
- PropertyDependencyInfo propertyDependency = new PropertyDependencyInfo(pi.getFilename(), pi.getName());
+ String type = ConfigHelper.fileNameToConfigType(pi.getFilename());
+ String name = pi.getName();
+ PropertyDependencyInfo propertyDependency =
+ new PropertyDependencyInfo(type, name);
if (dependedByMap.keySet().contains(pdi)) {
dependedByMap.get(pdi).add(propertyDependency);
} else {
- Set<PropertyDependencyInfo> newDependenciesSet = new HashSet<PropertyDependencyInfo>();
+ Set<PropertyDependencyInfo> newDependenciesSet =
+ new HashSet<PropertyDependencyInfo>();
newDependenciesSet.add(propertyDependency);
dependedByMap.put(pdi, newDependenciesSet);
}
@@ -612,7 +617,10 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
// Go through all service-configs again and set their 'depended-by' if necessary.
for (ServiceModule serviceModule : serviceModules.values()) {
for (PropertyInfo pi : serviceModule.getModuleInfo().getProperties()) {
- Set<PropertyDependencyInfo> set = dependedByMap.remove(new PropertyDependencyInfo(pi.getFilename(), pi.getName()));
+ String type = ConfigHelper.fileNameToConfigType(pi.getFilename());
+ String name = pi.getName();
+ Set<PropertyDependencyInfo> set =
+ dependedByMap.remove(new PropertyDependencyInfo(type, name));
if (set != null) {
pi.getDependedByProperties().addAll(set);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
index a99a132..bf20d6e 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
@@ -41,7 +41,7 @@
</value-attributes>
<depends-on>
<property>
- <type>mapred-site.xml</type>
+ <type>mapred-site</type>
<name>mapreduce.map.memory.mb</name>
</property>
</depends-on>
@@ -186,7 +186,7 @@
</value-attributes>
<depends-on>
<property>
- <type>yarn-site.xml</type>
+ <type>yarn-site</type>
<name>yarn.scheduler.maximum-allocation-mb</name>
</property>
</depends-on>
@@ -206,7 +206,7 @@
</value-attributes>
<depends-on>
<property>
- <type>yarn-site.xml</type>
+ <type>yarn-site</type>
<name>yarn.scheduler.maximum-allocation-mb</name>
</property>
</depends-on>
@@ -281,7 +281,7 @@
</value-attributes>
<depends-on>
<property>
- <type>yarn-site.xml</type>
+ <type>yarn-site</type>
<name>yarn.scheduler.maximum-allocation-mb</name>
</property>
</depends-on>
@@ -306,7 +306,7 @@
<display-name>MR AppMaster Java Heap Size</display-name>
<depends-on>
<property>
- <type>mapred-site.xml</type>
+ <type>mapred-site</type>
<name>yarn.app.mapreduce.am.resource.mb</name>
</property>
</depends-on>
@@ -328,7 +328,7 @@
<display-name>MR AppMaster Java Heap Size</display-name>
<depends-on>
<property>
- <type>mapred-site.xml</type>
+ <type>mapred-site</type>
<name>yarn.app.mapreduce.am.resource.mb</name>
</property>
</depends-on>
@@ -383,7 +383,7 @@
<display-name>MR Map Java Heap Size</display-name>
<depends-on>
<property>
- <type>mapred-site.xml</type>
+ <type>mapred-site</type>
<name>mapreduce.map.memory.mb</name>
</property>
</depends-on>
@@ -399,7 +399,7 @@
<display-name>MR Reduce Java Heap Size</display-name>
<depends-on>
<property>
- <type>mapred-site.xml</type>
+ <type>mapred-site</type>
<name>mapreduce.reduce.memory.mb</name>
</property>
</depends-on>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
index e6407ef..e71dfc9 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
@@ -80,7 +80,7 @@
</value-attributes>
<depends-on>
<property>
- <type>yarn-site.xml</type>
+ <type>yarn-site</type>
<name>yarn.nodemanager.resource.memory-mb</name>
</property>
</depends-on>
@@ -104,7 +104,7 @@
</value-attributes>
<depends-on>
<property>
- <type>yarn-site.xml</type>
+ <type>yarn-site</type>
<name>yarn.nodemanager.resource.memory-mb</name>
</property>
</depends-on>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/properties.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json
index b230cb7..94374bc 100644
--- a/ambari-server/src/main/resources/properties.json
+++ b/ambari-server/src/main/resources/properties.json
@@ -376,6 +376,7 @@
"recommend",
"hosts",
"services",
+ "changed_configurations",
"recommendations",
"recommendations/blueprint",
"recommendations/blueprint/configurations",
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/scripts/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/stack_advisor.py b/ambari-server/src/main/resources/scripts/stack_advisor.py
index 60f87f9..1d67872 100755
--- a/ambari-server/src/main/resources/scripts/stack_advisor.py
+++ b/ambari-server/src/main/resources/scripts/stack_advisor.py
@@ -26,9 +26,14 @@ import traceback
RECOMMEND_COMPONENT_LAYOUT_ACTION = 'recommend-component-layout'
VALIDATE_COMPONENT_LAYOUT_ACTION = 'validate-component-layout'
RECOMMEND_CONFIGURATIONS = 'recommend-configurations'
+RECOMMEND_CONFIGURATION_DEPENDENCIES = 'recommend-configuration-dependencies'
VALIDATE_CONFIGURATIONS = 'validate-configurations'
-ALL_ACTIONS = [ RECOMMEND_COMPONENT_LAYOUT_ACTION, VALIDATE_COMPONENT_LAYOUT_ACTION, RECOMMEND_CONFIGURATIONS, VALIDATE_CONFIGURATIONS ]
+ALL_ACTIONS = [RECOMMEND_COMPONENT_LAYOUT_ACTION,
+ VALIDATE_COMPONENT_LAYOUT_ACTION,
+ RECOMMEND_CONFIGURATIONS,
+ RECOMMEND_CONFIGURATION_DEPENDENCIES,
+ VALIDATE_CONFIGURATIONS]
USAGE = "Usage: <action> <hosts_file> <services_file>\nPossible actions are: {0}\n".format( str(ALL_ACTIONS) )
SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
@@ -101,6 +106,10 @@ def main(argv=None):
elif action == RECOMMEND_CONFIGURATIONS:
result = stackAdvisor.recommendConfigurations(services, hosts)
result_file = os.path.join(actionDir, "configurations.json")
+ elif action == RECOMMEND_CONFIGURATION_DEPENDENCIES:
+ result = stackAdvisor.recommendConfigurations(services, hosts)
+ result_file = os.path.join(actionDir, "configurations.json")
+ result = filterResult(result, services)
else: # action == VALIDATE_CONFIGURATIONS
result = stackAdvisor.validateConfigurations(services, hosts)
result_file = os.path.join(actionDir, "configurations-validation.json")
@@ -108,6 +117,45 @@ def main(argv=None):
dumpJson(result, result_file)
pass
+# returns recommendations only for changed and depended properties
+def filterResult(result, services):
+ allRequestedProperties = getAllRequestedProperties(services)
+
+ configs = result['recommendations']['blueprint']['configurations']
+ filteredConfigs = {}
+ for type, names in configs.items():
+ for name in names['properties']:
+ if type in allRequestedProperties.keys() and \
+ name in allRequestedProperties[type]:
+ if type not in filteredConfigs.keys():
+ filteredConfigs[type] = {'properties': {}}
+ filteredConfigs[type]['properties'][name] = \
+ configs[type]['properties'][name]
+ if 'property_attributes' in names.keys():
+ for name in names['property_attributes']:
+ if type in allRequestedProperties.keys() and \
+ name in allRequestedProperties[type]:
+ if type not in filteredConfigs.keys():
+ filteredConfigs[type] = {'property_Attributes': {}}
+ elif 'property_attributes' not in filteredConfigs[type].keys():
+ filteredConfigs[type]['property_attributes'] = {}
+ filteredConfigs[type]['property_attributes'][name] = \
+ configs[type]['property_attributes'][name]
+
+ result['recommendations']['blueprint']['configurations'] = filteredConfigs
+ return result
+
+def getAllRequestedProperties(services):
+ changedConfigs = []
+ changedConfigs.extend(services['changed-configurations'])
+ changedConfigs.extend(services['depended-configurations'])
+ allRequestedProperties = {}
+ for config in changedConfigs:
+ if config['type'] in allRequestedProperties:
+ allRequestedProperties[config['type']].append(config['name'])
+ else:
+ allRequestedProperties[config['type']] = [config['name']]
+ return allRequestedProperties
def instantiateStackAdvisor(stackName, stackVersion, parentVersions):
"""Instantiates StackAdvisor implementation for the specified Stack"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
index 853a732..53591cd 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
@@ -105,7 +105,7 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor):
putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m")
putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024))
- def getConfigurationClusterSummary(self, servicesList, hosts, components):
+ def getConfigurationClusterSummary(self, servicesList, hosts, components, services):
hBaseInstalled = False
if 'HBASE' in servicesList:
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index c9ded5e..2405954 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -88,23 +88,48 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
"AMBARI_METRICS": self.recommendAmsConfigurations
}
- def putProperty(self, config, configType):
- if configType not in config or "properties" not in config[configType]:
- config[configType] = {"properties": {}}
+ def putProperty(self, config, configType, services=None):
+ userConfigs = {}
+ changedConfigs = []
+ # if services parameter, prefer values, set by user
+ if services:
+ if 'configurations' in services.keys():
+ userConfigs = services['configurations']
+ if 'changed-configurations' in services.keys():
+ changedConfigs = services["changed-configurations"]
+
+ if configType not in config:
+ config[configType] = {}
+ if"properties" not in config[configType]:
+ config[configType]["properties"] = {}
def appendProperty(key, value):
- config[configType]["properties"][key] = str(value)
+ if {'type': configType, 'name': key} in changedConfigs:
+ config[configType]["properties"][key] = userConfigs[configType]['properties'][key]
+ else:
+ config[configType]["properties"][key] = str(value)
return appendProperty
+ def putPropertyAttribute(self, config, configType):
+ if configType not in config:
+ config[configType] = {}
+ def appendPropertyAttribute(key, attribute, attributeValue):
+ if"property_attributes" not in config[configType]:
+ config[configType]["property_attributes"] = {}
+ if key not in config[configType]["property_attributes"]:
+ config[configType]["property_attributes"][key] = {}
+ config[configType]["property_attributes"][key][attribute] = str(attributeValue)
+ return appendPropertyAttribute
+
def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
- putYarnProperty = self.putProperty(configurations, "yarn-site")
- putYarnEnvProperty = self.putProperty(configurations, "yarn-env")
+ putYarnProperty = self.putProperty(configurations, "yarn-site", services)
+ putYarnEnvProperty = self.putProperty(configurations, "yarn-env", services)
putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer'])))
putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer']))
putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer'])))
putYarnEnvProperty('min_user_id', self.get_system_min_uid())
def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts):
- putMapredProperty = self.putProperty(configurations, "mapred-site")
+ putMapredProperty = self.putProperty(configurations, "mapred-site", services)
putMapredProperty('yarn.app.mapreduce.am.resource.mb', int(clusterData['amMemory']))
putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(round(0.8 * clusterData['amMemory']))) + "m")
putMapredProperty('mapreduce.map.memory.mb', clusterData['mapMemory'])
@@ -184,7 +209,18 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
- def getConfigurationClusterSummary(self, servicesList, hosts, components):
+ def getHostWithComponent(self, serviceName, componentName, services, hosts):
+ if services is not None and hosts is not None:
+ service = [serviceEntry for serviceEntry in services["services"] if serviceEntry["StackServices"]["service_name"] == serviceName][0]
+ components = [componentEntry for componentEntry in service["components"] if componentEntry["StackServiceComponents"]["component_name"] == "NODEMANAGER"]
+ if (len(components) > 0 and len(components[0]["StackServiceComponents"]["hostnames"]) > 0):
+ # NodeManager available - determine hosts and memory
+ componentHostname = components[0]["StackServiceComponents"]["hostnames"][0]
+ componentHosts = [host for host in hosts["items"] if host["Hosts"]["host_name"] == componentHostname]
+ if (componentHosts is not None and len(componentHosts) > 0):
+ return componentHosts[0]
+
+ def getConfigurationClusterSummary(self, servicesList, hosts, components, services):
hBaseInstalled = False
if 'HBASE' in servicesList:
@@ -199,7 +235,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
}
if len(hosts["items"]) > 0:
- host = hosts["items"][0]["Hosts"]
+ nodeManagerHost = self.getHostWithComponent("YARN", "NODEMANAGER", services, hosts)
+ if nodeManagerHost is not None:
+ host = nodeManagerHost["Hosts"]
+ else:
+ host = hosts["items"][0]["Hosts"]
cluster["cpu"] = host["cpu_count"]
cluster["disk"] = len(host["disk_info"])
cluster["ram"] = int(host["total_mem"] / (1024 * 1024))
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
index 826f458..9137bdb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml
@@ -349,7 +349,7 @@
</value-attributes>
<depends-on>
<property>
- <type>yarn-site.xml</type>
+ <type>yarn-site</type>
<name>yarn.nodemanager.resource.cpu-vcores</name>
</property>
</depends-on>
@@ -367,7 +367,7 @@
</value-attributes>
<depends-on>
<property>
- <type>yarn-site.xml</type>
+ <type>yarn-site</type>
<name>yarn.nodemanager.resource.cpu-vcores</name>
</property>
</depends-on>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index b8ddc49..87748d2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -38,8 +38,19 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
super(HDP22StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts)
- putYarnProperty = self.putProperty(configurations, "yarn-site")
+ putYarnProperty = self.putProperty(configurations, "yarn-site", services)
putYarnProperty('yarn.nodemanager.resource.cpu-vcores', clusterData['cpu'])
+ # Property Attributes
+ putYarnPropertyAttribute = self.putPropertyAttribute(configurations, "yarn-site")
+ nodeManagerHost = self.getHostWithComponent("YARN", "NODEMANAGER", services, hosts)
+ if (nodeManagerHost is not None):
+ putYarnProperty('yarn.nodemanager.resource.cpu-vcores', nodeManagerHost["Hosts"]["cpu_count"] * 2)
+ putYarnPropertyAttribute('yarn.nodemanager.resource.memory-mb', 'max', int(nodeManagerHost["Hosts"]["total_mem"] / 1024)) # total_mem in kb
+ putYarnPropertyAttribute('yarn.nodemanager.resource.cpu-vcores', 'max', nodeManagerHost["Hosts"]["cpu_count"] * 4)
+ putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-vcores', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"])
+ putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-vcores', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"])
+ putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
+ putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
putHdfsProperty = self.putProperty(configurations, "hdfs-site")
@@ -156,14 +167,25 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
return self.toConfigurationValidationProblems(validationItems, "tez-site")
def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts):
- putMapredProperty = self.putProperty(configurations, "mapred-site")
+ self.recommendYARNConfigurations(configurations, clusterData, services, hosts)
+ putMapredProperty = self.putProperty(configurations, "mapred-site", services)
putMapredProperty('yarn.app.mapreduce.am.resource.mb', int(clusterData['amMemory']))
putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(round(0.8 * clusterData['amMemory']))) + "m" + " -Dhdp.version=${hdp.version}")
- putMapredProperty('mapreduce.map.memory.mb', clusterData['mapMemory'])
+ putMapredProperty('mapreduce.map.memory.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
putMapredProperty('mapreduce.reduce.memory.mb', int(clusterData['reduceMemory']))
putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m")
putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m")
putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024))
+ # Property Attributes
+ putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site")
+ yarnMinAllocationSize = int(clusterData['ramPerContainer'])
+ yarnMaxAllocationSize = max(30 * int(clusterData['ramPerContainer']), configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])
+ putMapredPropertyAttribute("mapreduce.map.memory.mb", "max", yarnMaxAllocationSize)
+ putMapredPropertyAttribute("mapreduce.map.memory.mb", "min", yarnMinAllocationSize)
+ putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "max", yarnMaxAllocationSize)
+ putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "min", yarnMinAllocationSize)
+ putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "max", yarnMaxAllocationSize)
+ putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "min", yarnMinAllocationSize)
def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
http://git-wip-us.apache.org/repos/asf/ambari/blob/d7fcd258/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
index cf35e47..3353af1 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
@@ -133,7 +133,7 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor):
"-server -Xmx" + str(int(0.8 * clusterData["amMemory"]))
+ "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC")
- def getConfigurationClusterSummary(self, servicesList, hosts, components):
+ def getConfigurationClusterSummary(self, servicesList, hosts, components, services):
hBaseInstalled = False
if 'HBASE' in servicesList: