You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2014/08/08 01:47:17 UTC
git commit: AMBARI-6781. BE: Provide configurations recommendations
via /recommendations endpoint on stack-version
Repository: ambari
Updated Branches:
refs/heads/trunk e8b355412 -> b13bb71dc
AMBARI-6781. BE: Provide configurations recommendations via /recommendations endpoint on stack-version
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b13bb71d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b13bb71d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b13bb71d
Branch: refs/heads/trunk
Commit: b13bb71dc27494f220f97f1b5c6ba8d05354a55e
Parents: e8b3554
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Thu Aug 7 14:23:50 2014 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Thu Aug 7 16:47:08 2014 -0700
----------------------------------------------------------------------
.../stackadvisor/StackAdvisorHelper.java | 21 +++
.../stackadvisor/StackAdvisorHelper.java.orig | 116 ++++++++++++++
.../stackadvisor/StackAdvisorRequest.java | 22 +++
...GetComponentLayoutRecommnedationCommand.java | 5 +
.../GetComponentLayoutValidationCommand.java | 5 +
.../GetConfigurationRecommnedationCommand.java | 118 ++++++++++++++
.../commands/StackAdvisorCommand.java | 5 +-
.../internal/StackAdvisorResourceProvider.java | 75 ++++-----
.../src/main/resources/scripts/stack_advisor.py | 20 +--
.../stacks/HDP/2.0.6/services/stack_advisor.py | 159 ++++++++++++++++++-
.../stacks/HDP/2.1/services/stack_advisor.py | 41 +++++
.../main/resources/stacks/HDP/stack_advisor.py | 2 +-
.../commands/StackAdvisorCommandTest.java | 5 +
13 files changed, 544 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
index 213b0f0..156b637 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType;
import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutRecommnedationCommand;
import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutValidationCommand;
+import org.apache.ambari.server.api.services.stackadvisor.commands.GetConfigurationRecommnedationCommand;
import org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand;
import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
@@ -105,6 +106,9 @@ public class StackAdvisorHelper {
if (requestType == StackAdvisorRequestType.HOST_GROUPS) {
command = new GetComponentLayoutRecommnedationCommand(recommendationsDir, stackAdvisorScript,
requestId, saRunner);
+ } else if (requestType == StackAdvisorRequestType.CONFIGURATIONS) {
+ command = new GetConfigurationRecommnedationCommand(recommendationsDir, stackAdvisorScript,
+ requestId, saRunner);
} else {
throw new StackAdvisorException(String.format("Unsupported request type, type=%s",
requestType));
@@ -113,4 +117,21 @@ public class StackAdvisorHelper {
return command;
}
+ /**
+ * Return configurations recommendation based on hosts and services
+ * information.
+ *
+ * @param request the recommendation request
+ * @return {@link RecommendationResponse} instance
+ * @throws StackAdvisorException in case of stack advisor script errors
+ */
+ public synchronized RecommendationResponse getConfigurationRecommnedation(
+ StackAdvisorRequest request) throws StackAdvisorException {
+ requestId += 1;
+
+ GetConfigurationRecommnedationCommand command = new GetConfigurationRecommnedationCommand(
+ recommendationsDir, stackAdvisorScript, requestId, saRunner);
+ return command.invoke(request);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java.orig
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java.orig b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java.orig
new file mode 100644
index 0000000..213b0f0
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java.orig
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.api.services.stackadvisor;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType;
+import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutRecommnedationCommand;
+import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutValidationCommand;
+import org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand;
+import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
+import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
+import org.apache.ambari.server.configuration.Configuration;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+@Singleton
+public class StackAdvisorHelper {
+
+ private File recommendationsDir;
+ private String stackAdvisorScript;
+
+ /* Monotonically increasing requestid */
+ private int requestId = 0;
+ private StackAdvisorRunner saRunner;
+
+ @Inject
+ public StackAdvisorHelper(Configuration conf, StackAdvisorRunner saRunner) throws IOException {
+ this.recommendationsDir = conf.getRecommendationsDir();
+ this.stackAdvisorScript = conf.getStackAdvisorScript();
+ this.saRunner = saRunner;
+ }
+
+ /**
+ * Returns validation (component-layout or configurations) result for the
+ * request.
+ *
+ * @param validationRequest the validation request
+ * @return {@link ValidationResponse} instance
+ * @throws StackAdvisorException in case of stack advisor script errors
+ */
+ public synchronized ValidationResponse validate(StackAdvisorRequest request)
+ throws StackAdvisorException {
+ requestId += 1;
+
+ StackAdvisorCommand<ValidationResponse> command = createValidationCommand(request
+ .getRequestType());
+
+ return command.invoke(request);
+ }
+
+ StackAdvisorCommand<ValidationResponse> createValidationCommand(
+ StackAdvisorRequestType requestType) throws StackAdvisorException {
+ StackAdvisorCommand<ValidationResponse> command;
+ if (requestType == StackAdvisorRequestType.HOST_GROUPS) {
+ command = new GetComponentLayoutValidationCommand(recommendationsDir, stackAdvisorScript,
+ requestId, saRunner);
+ } else {
+ throw new StackAdvisorException(String.format("Unsupported request type, type=%s",
+ requestType));
+ }
+
+ return command;
+ }
+
+ /**
+ * Returns recommendation (component-layout or configurations) based on the
+ * request.
+ *
+ * @param request the recommendation request
+ * @return {@link RecommendationResponse} instance
+ * @throws StackAdvisorException in case of stack advisor script errors
+ */
+ public synchronized RecommendationResponse recommend(StackAdvisorRequest request)
+ throws StackAdvisorException {
+ requestId += 1;
+
+ StackAdvisorCommand<RecommendationResponse> command = createRecommendationCommand(request
+ .getRequestType());
+
+ return command.invoke(request);
+ }
+
+ StackAdvisorCommand<RecommendationResponse> createRecommendationCommand(
+ StackAdvisorRequestType requestType) throws StackAdvisorException {
+ StackAdvisorCommand<RecommendationResponse> command;
+ if (requestType == StackAdvisorRequestType.HOST_GROUPS) {
+ command = new GetComponentLayoutRecommnedationCommand(recommendationsDir, stackAdvisorScript,
+ requestId, saRunner);
+ } else {
+ throw new StackAdvisorException(String.format("Unsupported request type, type=%s",
+ requestType));
+ }
+
+ return command;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
index b82047e..bc1678e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
@@ -27,6 +27,8 @@ import java.util.Set;
import org.apache.commons.lang.StringUtils;
+import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.*;
+
/**
* Stack advisor request.
*/
@@ -38,6 +40,8 @@ public class StackAdvisorRequest {
private List<String> hosts = new ArrayList<String>();
private List<String> services = new ArrayList<String>();
private Map<String, Set<String>> componentHostsMap = new HashMap<String, Set<String>>();
+ private Map<String, Set<String>> hostComponents = new HashMap<String, Set<String>>();
+ private Map<String, Set<String>> hostGroupBindings = new HashMap<String, Set<String>>();
public String getStackName() {
return stackName;
@@ -71,6 +75,14 @@ public class StackAdvisorRequest {
return StringUtils.join(services, ",");
}
+ public Map<String, Set<String>> getHostComponents() {
+ return hostComponents;
+ }
+
+ public Map<String, Set<String>> getHostGroupBindings() {
+ return hostGroupBindings;
+ }
+
private StackAdvisorRequest(String stackName, String stackVersion) {
this.stackName = stackName;
this.stackVersion = stackVersion;
@@ -108,6 +120,16 @@ public class StackAdvisorRequest {
return this;
}
+ public StackAdvisorRequestBuilder forHostComponents(Map<String, Set<String>> hostComponents) {
+ this.instance.hostComponents = hostComponents;
+ return this;
+ }
+
+ public StackAdvisorRequestBuilder forHostsGroupBindings(Map<String, Set<String>> hostGroupBindings) {
+ this.instance.hostGroupBindings = hostGroupBindings;
+ return this;
+ }
+
public StackAdvisorRequest build() {
return this.instance;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java
index b3b5057..6e7533a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutRecommnedationCommand.java
@@ -51,6 +51,11 @@ public class GetComponentLayoutRecommnedationCommand extends
}
@Override
+ protected RecommendationResponse updateResponse(StackAdvisorRequest request, RecommendationResponse response) {
+ return response;
+ }
+
+ @Override
protected String getResultFileName() {
return "component-layout.json";
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java
index ac24d07..a5453f6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetComponentLayoutValidationCommand.java
@@ -50,6 +50,11 @@ public class GetComponentLayoutValidationCommand extends StackAdvisorCommand<Val
}
@Override
+ protected ValidationResponse updateResponse(StackAdvisorRequest request, ValidationResponse response) {
+ return response;
+ }
+
+ @Override
protected String getResultFileName() {
return "component-layout-validation.json";
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
new file mode 100644
index 0000000..48946aa
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.api.services.stackadvisor.commands;
+
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
+import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
+import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.*;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * {@link org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand} implementation for
+ * configuration recommendation.
+ */
+public class GetConfigurationRecommnedationCommand extends
+ StackAdvisorCommand<RecommendationResponse> {
+
+ public GetConfigurationRecommnedationCommand(File recommendationsDir,
+ String stackAdvisorScript, int requestId, StackAdvisorRunner saRunner) {
+ super(recommendationsDir, stackAdvisorScript, requestId, saRunner);
+ }
+
+ @Override
+ protected StackAdvisorCommandType getCommandType() {
+ return StackAdvisorCommandType.RECOMMEND_CONFIGURATIONS;
+ }
+
+ @Override
+ protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
+ if (request.getHosts().isEmpty() || request.getServices().isEmpty()) {
+ throw new StackAdvisorException("Hosts and services must not be empty");
+ }
+ }
+
+ @Override
+ protected StackAdvisorData adjust(StackAdvisorData data, StackAdvisorRequest request) {
+ // do nothing
+ return data;
+ }
+
+ @Override
+ protected RecommendationResponse updateResponse(StackAdvisorRequest request, RecommendationResponse response) {
+ response.getRecommendations().getBlueprint().setHostGroups(processHostGroups(request));
+ response.getRecommendations().getBlueprintClusterBinding().setHostGroups(processHostGroupBindings(request));
+ return response;
+ }
+
+ private Set<HostGroup> processHostGroups(StackAdvisorRequest request) {
+
+ Set<HostGroup> resultSet = new HashSet<HostGroup>();
+ for (Map.Entry<String, Set<String>> componentHost : request.getComponentHostsMap().entrySet()) {
+ String hostGroupName = componentHost.getKey();
+ Set<String> components = componentHost.getValue();
+ if (hostGroupName != null && components != null) {
+ HostGroup hostGroup = new HostGroup();
+ Set<Map<String, String>> componentsSet = new HashSet<Map<String, String>>();
+ for (String component : components) {
+ Map<String, String> componentMap = new HashMap<String, String>();
+ componentMap.put("name", component);
+ componentsSet.add(componentMap);
+ }
+ hostGroup.setComponents(componentsSet);
+ hostGroup.setName(hostGroupName);
+ resultSet.add(hostGroup);
+ }
+ }
+ return resultSet;
+ }
+
+ private Set<BindingHostGroup> processHostGroupBindings(StackAdvisorRequest request) {
+ Set<BindingHostGroup> resultSet = new HashSet<BindingHostGroup>();
+ for (Map.Entry<String, Set<String>> hostBinding : request.getHostGroupBindings().entrySet()) {
+ String hostGroupName = hostBinding.getKey();
+ Set<String> hosts = hostBinding.getValue();
+ if (hostGroupName != null && hosts != null) {
+ BindingHostGroup bindingHostGroup = new BindingHostGroup();
+ Set<Map<String, String>> hostsSet = new HashSet<Map<String, String>>();
+ for (String host : hosts) {
+ Map<String, String> hostMap = new HashMap<String, String>();
+ hostMap.put("name", host);
+ hostsSet.add(hostMap);
+ }
+ bindingHostGroup.setHosts(hostsSet);
+ bindingHostGroup.setName(hostGroupName);
+ resultSet.add(bindingHostGroup);
+ }
+ }
+ return resultSet;
+ }
+
+ @Override
+ protected String getResultFileName() {
+ return "configurations.json";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
index a9ff24b..1a30d38 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
@@ -193,7 +193,8 @@ public abstract class StackAdvisorCommand<T> extends BaseService {
String result = FileUtils.readFileToString(new File(requestDirectory, getResultFileName()));
- return this.mapper.readValue(result, this.type);
+ T response = this.mapper.readValue(result, this.type);
+ return updateResponse(request, response);
} catch (Exception e) {
String message = "Error occured during stack advisor command invocation";
LOG.warn(message, e);
@@ -201,6 +202,8 @@ public abstract class StackAdvisorCommand<T> extends BaseService {
}
}
+ protected abstract T updateResponse(StackAdvisorRequest request, T response);
+
/**
* Create request id directory for each call
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
index 9cf387a..95024d2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
@@ -93,11 +93,13 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
*/
List<String> hosts = (List<String>) getRequestProperty(request, HOST_PROPERTY);
List<String> services = (List<String>) getRequestProperty(request, SERVICES_PROPERTY);
- Map<String, Set<String>> componentHostsMap = calculateComponentHostsMap(request);
+ Map<String, Set<String>> hgComponentsMap = calculateHostGroupComponentsMap(request);
+ Map<String, Set<String>> hgHostsMap = calculateHostGroupHostsMap(request);
+ Map<String, Set<String>> componentHostsMap = calculateComponentHostsMap(hgComponentsMap, hgHostsMap);
StackAdvisorRequest saRequest = StackAdvisorRequestBuilder.forStack(stackName, stackVersion)
- .ofType(requestType).forHosts(hosts).forServices(services)
- .withComponentHostsMap(componentHostsMap).build();
+ .ofType(requestType).forHosts(hosts).forServices(services).forHostComponents(hgComponentsMap)
+ .forHostsGroupBindings(hgHostsMap).withComponentHostsMap(componentHostsMap).build();
return saRequest;
} catch (Exception e) {
@@ -113,25 +115,28 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
* Will prepare host-group names to components names map from the
* recommendation blueprint host groups.
*
- * @param hostGroups the blueprint host groups
+ * @param request stack advisor request
* @return host-group to components map
*/
@SuppressWarnings("unchecked")
- private Map<String, Set<String>> calculateHostGroupComponentsMap(
- Set<Map<String, Object>> hostGroups) {
+ private Map<String, Set<String>> calculateHostGroupComponentsMap(Request request) {
+ Set<Map<String, Object>> hostGroups =
+ (Set<Map<String, Object>>) getRequestProperty(request, BLUEPRINT_HOST_GROUPS_PROPERTY);
Map<String, Set<String>> map = new HashMap<String, Set<String>>();
- for (Map<String, Object> hostGroup : hostGroups) {
- String hostGroupName = (String) hostGroup.get(BLUEPRINT_HOST_GROUPS_NAME_PROPERTY);
+ if (hostGroups != null) {
+ for (Map<String, Object> hostGroup : hostGroups) {
+ String hostGroupName = (String) hostGroup.get(BLUEPRINT_HOST_GROUPS_NAME_PROPERTY);
- Set<Map<String, Object>> componentsSet = (Set<Map<String, Object>>) hostGroup
- .get(BLUEPRINT_HOST_GROUPS_COMPONENTS_PROPERTY);
+ Set<Map<String, Object>> componentsSet = (Set<Map<String, Object>>) hostGroup
+ .get(BLUEPRINT_HOST_GROUPS_COMPONENTS_PROPERTY);
- Set<String> components = new HashSet<String>();
- for (Map<String, Object> component : componentsSet) {
- components.add((String) component.get(BLUEPRINT_HOST_GROUPS_COMPONENTS_NAME_PROPERTY));
- }
+ Set<String> components = new HashSet<String>();
+ for (Map<String, Object> component : componentsSet) {
+ components.add((String) component.get(BLUEPRINT_HOST_GROUPS_COMPONENTS_NAME_PROPERTY));
+ }
- map.put(hostGroupName, components);
+ map.put(hostGroupName, components);
+ }
}
return map;
@@ -141,52 +146,48 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
* Will prepare host-group names to hosts names map from the recommendation
* binding host groups.
*
- * @param bindingHostGroups the binding host groups
+ * @param request stack advisor request
* @return host-group to hosts map
*/
@SuppressWarnings("unchecked")
- private Map<String, Set<String>> calculateHostGroupHostsMap(
- Set<Map<String, Object>> bindingHostGroups) {
+ private Map<String, Set<String>> calculateHostGroupHostsMap(Request request) {
+ Set<Map<String, Object>> bindingHostGroups =
+ (Set<Map<String, Object>>) getRequestProperty(request, BINDING_HOST_GROUPS_PROPERTY);
Map<String, Set<String>> map = new HashMap<String, Set<String>>();
+ if (bindingHostGroups != null) {
+ for (Map<String, Object> hostGroup : bindingHostGroups) {
+ String hostGroupName = (String) hostGroup.get(BINDING_HOST_GROUPS_NAME_PROPERTY);
- for (Map<String, Object> hostGroup : bindingHostGroups) {
- String hostGroupName = (String) hostGroup.get(BINDING_HOST_GROUPS_NAME_PROPERTY);
+ Set<Map<String, Object>> hostsSet = (Set<Map<String, Object>>) hostGroup
+ .get(BINDING_HOST_GROUPS_HOSTS_PROPERTY);
- Set<Map<String, Object>> hostsSet = (Set<Map<String, Object>>) hostGroup
- .get(BINDING_HOST_GROUPS_HOSTS_PROPERTY);
+ Set<String> hosts = new HashSet<String>();
+ for (Map<String, Object> host : hostsSet) {
+ hosts.add((String) host.get(BINDING_HOST_GROUPS_HOSTS_NAME_PROPERTY));
+ }
- Set<String> hosts = new HashSet<String>();
- for (Map<String, Object> host : hostsSet) {
- hosts.add((String) host.get(BINDING_HOST_GROUPS_HOSTS_NAME_PROPERTY));
+ map.put(hostGroupName, hosts);
}
-
- map.put(hostGroupName, hosts);
}
return map;
}
@SuppressWarnings("unchecked")
- private Map<String, Set<String>> calculateComponentHostsMap(Request request) {
+ private Map<String, Set<String>> calculateComponentHostsMap(Map<String, Set<String>> hostGroups,
+ Map<String, Set<String>> bindingHostGroups) {
/*
* ClassCastException may occur in case of body inconsistency: property
* missed, etc.
*/
- Set<Map<String, Object>> hostGroups = (Set<Map<String, Object>>) getRequestProperty(request,
- BLUEPRINT_HOST_GROUPS_PROPERTY);
- Set<Map<String, Object>> bindingHostGroups = (Set<Map<String, Object>>) getRequestProperty(
- request, BINDING_HOST_GROUPS_PROPERTY);
Map<String, Set<String>> componentHostsMap = new HashMap<String, Set<String>>();
if (null != bindingHostGroups && null != hostGroups) {
- Map<String, Set<String>> hgComponentsMap = calculateHostGroupComponentsMap(hostGroups);
- Map<String, Set<String>> hgHostsMap = calculateHostGroupHostsMap(bindingHostGroups);
-
- for (Map.Entry<String, Set<String>> hgComponents : hgComponentsMap.entrySet()) {
+ for (Map.Entry<String, Set<String>> hgComponents : hostGroups.entrySet()) {
String hgName = hgComponents.getKey();
Set<String> components = hgComponents.getValue();
- Set<String> hosts = hgHostsMap.get(hgName);
+ Set<String> hosts = bindingHostGroups.get(hgName);
for (String component : components) {
Set<String> componentHosts = componentHostsMap.get(component);
if (componentHosts == null) { // if was not initialized
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/resources/scripts/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/stack_advisor.py b/ambari-server/src/main/resources/scripts/stack_advisor.py
index 1eb5e4d..05d3a20 100755
--- a/ambari-server/src/main/resources/scripts/stack_advisor.py
+++ b/ambari-server/src/main/resources/scripts/stack_advisor.py
@@ -82,7 +82,8 @@ def main(argv=None):
stackName = services["Versions"]["stack_name"]
stackVersion = services["Versions"]["stack_version"]
parentVersions = []
- if "parent_stack_version" in services["Versions"]:
+ if "parent_stack_version" in services["Versions"] and \
+ services["Versions"]["parent_stack_version"] is not None:
parentVersions = [ services["Versions"]["parent_stack_version"] ]
stackAdvisor = instantiateStackAdvisor(stackName, stackVersion, parentVersions)
@@ -120,22 +121,23 @@ def instantiateStackAdvisor(stackName, stackVersion, parentVersions):
versions = [stackVersion]
versions.extend(parentVersions)
- for version in versions:
+ for version in reversed(versions):
try:
path = STACK_ADVISOR_IMPL_PATH_TEMPLATE.format(stackName, version)
className = STACK_ADVISOR_IMPL_CLASS_TEMPLATE.format(stackName, version.replace('.', ''))
with open(path, 'rb') as fp:
- stack_advisor_impl = imp.load_module( 'stack_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE) )
- clazz = getattr(stack_advisor_impl, className)
-
- print "StackAdvisor for stack {0}, version {1} will be used".format(stackName, version)
- return clazz();
+ stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE))
+ print "StackAdvisor implementation for stack {0}, version {1} was loaded".format(stackName, version)
except Exception, e:
print "StackAdvisor implementation for stack {0}, version {1} was not found".format(stackName, version)
- print "StackAdvisor default implementation will be used!"
- return stack_advisor.StackAdvisor()
+ try:
+ clazz = getattr(stack_advisor_impl, className)
+ return clazz()
+ except Exception, e:
+ print "Returning default implementation"
+ return stack_advisor.StackAdvisor()
if __name__ == '__main__':
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 85c3a8e..402d95b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -204,8 +204,163 @@ class HDP206StackAdvisor(StackAdvisor):
pass
def recommendConfigurations(self, services, hosts):
- """Returns Services object with configurations object populated"""
- pass
+ stackName = services["Versions"]["stack_name"]
+ stackVersion = services["Versions"]["stack_version"]
+ hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ components = [component["StackServiceComponents"]["component_name"]
+ for service in services["services"]
+ for component in service["components"]]
+
+ clusterData = self.getClusterData(servicesList, hosts, components)
+
+ recommendations = {
+ "Versions": {"stack_name": stackName, "stack_version": stackVersion},
+ "hosts": hostsList,
+ "services": servicesList,
+ "recommendations": {
+ "blueprint": {
+ "configurations": {},
+ "host_groups": []
+ },
+ "blueprint_cluster_binding": {
+ "host_groups": []
+ }
+ }
+ }
+
+ configurations = recommendations["recommendations"]["blueprint"]["configurations"]
+
+ for service in servicesList:
+ calculation = self.recommendServiceConfigurations(service)
+ if calculation is not None:
+ calculation(configurations, clusterData)
+
+ return recommendations
+
+ def recommendServiceConfigurations(self, service):
+ return {
+ "YARN": self.recommendYARNConfigurations,
+ "MAPREDUCE2": self.recommendMapReduce2Configurations,
+ "HIVE": self.recommendHiveConfigurations,
+ "OOZIE": self.recommendOozieConfigurations
+ }.get(service, None)
+
+ def putProperty(self, config, configType):
+ config[configType] = {"properties": {}}
+ def appendProperty(key, value):
+ config[configType]["properties"][key] = str(value)
+ return appendProperty
+
+ def recommendYARNConfigurations(self, configurations, clusterData):
+ putYarnProperty = self.putProperty(configurations, "yarn-site")
+ putYarnProperty('yarn.nodemanager.resource.memory-mb', clusterData['containers'] * clusterData['ramPerContainer'])
+ putYarnProperty('yarn.scheduler.minimum-allocation-mb', clusterData['ramPerContainer'])
+ putYarnProperty('yarn.scheduler.maximum-allocation-mb', clusterData['containers'] * clusterData['ramPerContainer'])
+
+ def recommendHiveConfigurations(self, configurations, clusterData):
+ containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else clusterData['reduceMemory']
+ containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize)
+ putHiveProperty = self.putProperty(configurations, "hive-site")
+ putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(containerSize / 3) * 1048576)
+ putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(0.8 * containerSize))
+ + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC")
+ putHiveProperty('hive.tez.container.size', containerSize)
+
+ def recommendMapReduce2Configurations(self, configurations, clusterData):
+ putMapredProperty = self.putProperty(configurations, "mapred-site")
+ putMapredProperty('yarn.app.mapreduce.am.resource.mb', clusterData['amMemory'])
+ putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * clusterData['amMemory'])) + "m")
+ putMapredProperty('mapreduce.map.memory.mb', clusterData['mapMemory'])
+ putMapredProperty('mapreduce.reduce.memory.mb', clusterData['reduceMemory'])
+ putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(0.8 * clusterData['mapMemory'])) + "m")
+ putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8 * clusterData['reduceMemory'])) + "m")
+ putMapredProperty('mapreduce.task.io.sort.mb', int(min(0.4 * clusterData['mapMemory'], 1024)))
+
+ def recommendOozieConfigurations(self, configurations, clusterData):
+ if "FALCON_SERVER" in clusterData["components"]:
+ putMapredProperty = self.putProperty(configurations, "oozie-site")
+ putMapredProperty("oozie.services.ext",
+ "org.apache.oozie.service.JMSAccessorService," +
+ "org.apache.oozie.service.PartitionDependencyManagerService," +
+ "org.apache.oozie.service.HCatAccessorService")
+
+ def getClusterData(self, servicesList, hosts, components):
+
+ hBaseInstalled = False
+ if 'HBASE' in servicesList:
+ hBaseInstalled = True
+
+ cluster = {
+ "cpu": 0,
+ "disk": 0,
+ "ram": 0,
+ "hBaseInstalled": hBaseInstalled,
+ "components": components
+ }
+
+ if len(hosts["items"]) > 0:
+ host = hosts["items"][0]["Hosts"]
+ cluster["cpu"] = host["cpu_count"]
+ cluster["disk"] = len(host["disk_info"])
+ cluster["ram"] = int(host["total_mem"] / (1024 * 1024))
+
+ ramRecommendations = [
+ {"os":1, "hbase":1},
+ {"os":2, "hbase":1},
+ {"os":2, "hbase":2},
+ {"os":4, "hbase":4},
+ {"os":6, "hbase":8},
+ {"os":8, "hbase":8},
+ {"os":8, "hbase":8},
+ {"os":12, "hbase":16},
+ {"os":24, "hbase":24},
+ {"os":32, "hbase":32},
+ {"os":64, "hbase":64}
+ ]
+ index = {
+ cluster["ram"] <= 4: 0,
+ 4 < cluster["ram"] <= 8: 1,
+ 8 < cluster["ram"] <= 16: 2,
+ 16 < cluster["ram"] <= 24: 3,
+ 24 < cluster["ram"] <= 48: 4,
+ 48 < cluster["ram"] <= 64: 5,
+ 64 < cluster["ram"] <= 72: 6,
+ 72 < cluster["ram"] <= 96: 7,
+ 96 < cluster["ram"] <= 128: 8,
+ 128 < cluster["ram"] <= 256: 9,
+ 256 < cluster["ram"]: 10
+ }[1]
+ cluster["reservedRam"] = ramRecommendations[index]["os"]
+ cluster["hbaseRam"] = ramRecommendations[index]["hbase"]
+
+ cluster["minContainerSize"] = {
+ cluster["ram"] <= 4: 256,
+ 4 < cluster["ram"] <= 8: 512,
+ 8 < cluster["ram"] <= 24: 1024,
+ 24 < cluster["ram"]: 2048
+ }[1]
+
+ '''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) / MIN_CONTAINER_SIZE))))'''
+ cluster["containers"] = max(3,
+ min(2 * cluster["cpu"],
+ int(min(1.8 * cluster["disk"],
+ cluster["ram"] / cluster["minContainerSize"]))))
+
+ '''ramPerContainers = max(2GB, RAM - reservedRam - hBaseRam) / containers'''
+ cluster["ramPerContainer"] = max(2048,
+ cluster["ram"] - cluster["reservedRam"] - cluster["hbaseRam"])
+ cluster["ramPerContainer"] /= cluster["containers"]
+ '''If greater than 1GB, value will be in multiples of 512.'''
+ if cluster["ramPerContainer"] > 1024:
+ cluster["ramPerContainer"] = ceil(cluster["ramPerContainer"] / 512) * 512
+
+ cluster["mapMemory"] = int(cluster["ramPerContainer"])
+ cluster["reduceMemory"] = cluster["ramPerContainer"]
+ cluster["amMemory"] = max(cluster["mapMemory"], cluster["reduceMemory"])
+
+ return cluster
+
def validateConfigurations(self, services, hosts):
"""Returns array of Validation objects about issues with configuration values provided in services"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
new file mode 100644
index 0000000..eb2faa1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import socket
+
+from stack_advisor import StackAdvisor
+
+class HDP21StackAdvisor(HDP206StackAdvisor):
+
+ def recommendServiceConfigurations(self, service):
+ calculator = super(HDP21StackAdvisor, self).recommendServiceConfigurations(service)
+ if calculator is None:
+ return {
+ "TEZ": self.recommendTezConfigurations
+ }.get(service, None)
+ else:
+ return calculator
+
+ def recommendTezConfigurations(self, configurations, clusterData):
+ putTezProperty = self.putProperty(configurations, "tez-site")
+ putTezProperty("tez.am.resource.memory.mb", clusterData['amMemory'])
+ putTezProperty("tez.am.java.opts",
+ "-server -Xmx" + str(int(0.8 * clusterData["amMemory"]))
+ + "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC")
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py
index fdef482..5f66fff 100644
--- a/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py
@@ -17,7 +17,7 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-class StackAdvisor():
+class StackAdvisor(object):
def recommendComponentLayout(self, services, hosts):
"""Returns Services object with hostnames array populated for components"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/b13bb71d/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
index 90dafcb..5e4e3d0 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommandTest.java
@@ -185,6 +185,11 @@ public class StackAdvisorCommandTest {
protected StackAdvisorCommandType getCommandType() {
return StackAdvisorCommandType.RECOMMEND_COMPONENT_LAYOUT;
}
+
+ @Override
+ protected TestResource updateResponse(StackAdvisorRequest request, TestResource response) {
+ return response;
+ }
}
public static class TestResource {