You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2017/01/19 06:36:45 UTC
ambari git commit: AMBARI-19546. Add support for call context to the
stack advisor (smohanty)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 0b8f8c890 -> 9ef624fe8
AMBARI-19546. Add support for call context to the stack advisor (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9ef624fe
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9ef624fe
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9ef624fe
Branch: refs/heads/branch-2.5
Commit: 9ef624fe871f8dfdeb89c2b0e3f1e9b1f51396ea
Parents: 0b8f8c8
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Jan 18 22:36:30 2017 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Jan 18 22:36:35 2017 -0800
----------------------------------------------------------------------
.../StackAdvisorBlueprintProcessor.java | 8 +++++
.../stackadvisor/StackAdvisorRequest.java | 15 +++++++++
.../commands/StackAdvisorCommand.java | 4 +++
.../internal/StackAdvisorResourceProvider.java | 31 +++++++++++++++++--
.../src/main/resources/properties.json | 3 ++
.../src/main/resources/scripts/stack_advisor.py | 14 ++++++---
.../stacks/HDP/2.0.6/services/stack_advisor.py | 25 ++++++++++++---
.../src/main/resources/stacks/stack_advisor.py | 11 +++++++
.../StackAdvisorResourceProviderTest.java | 32 ++++++++++++++++++++
.../stacks/2.0.6/common/test_stack_advisor.py | 8 ++---
10 files changed, 135 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
index 67dfded..eb3d7e2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
@@ -66,6 +66,13 @@ public class StackAdvisorBlueprintProcessor {
stackAdvisorHelper = instance;
}
+ private static final Map<String, String> userContext;
+ static
+ {
+ userContext = new HashMap<String, String>();
+ userContext.put("operation", "ClusterCreate");
+ }
+
/**
* Recommend configurations by the stack advisor, then store the results in cluster topology.
* @param clusterTopology cluster topology instance
@@ -97,6 +104,7 @@ public class StackAdvisorBlueprintProcessor {
.forHostComponents(gatherHostGroupComponents(clusterTopology))
.withComponentHostsMap(componentHostsMap)
.withConfigurations(calculateConfigs(clusterTopology))
+ .withUserContext(userContext)
.ofType(requestType)
.build();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
index b804c7e..c655816 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
@@ -47,6 +47,7 @@ public class StackAdvisorRequest {
private Map<String, Map<String, Map<String, String>>> configurations = new HashMap<String, Map<String, Map<String, String>>>();
private List<ChangedConfigInfo> changedConfigurations = new LinkedList<ChangedConfigInfo>();
private Set<RecommendationResponse.ConfigGroup> configGroups;
+ private Map<String, String> userContext = new HashMap<String, String>();
public String getStackName() {
return stackName;
@@ -100,6 +101,14 @@ public class StackAdvisorRequest {
this.changedConfigurations = changedConfigurations;
}
+ public Map<String, String> getUserContext() {
+ return this.userContext;
+ }
+
+ public void setUserContext(Map<String, String> userContext) {
+ this.userContext = userContext;
+ }
+
public Set<RecommendationResponse.ConfigGroup> getConfigGroups() {
return configGroups;
}
@@ -168,6 +177,12 @@ public class StackAdvisorRequest {
return this;
}
+ public StackAdvisorRequestBuilder withUserContext(
+ Map<String, String> userContext) {
+ this.instance.userContext = userContext;
+ return this;
+ }
+
public StackAdvisorRequestBuilder withConfigGroups(
Set<RecommendationResponse.ConfigGroup> configGroups) {
this.instance.configGroups = configGroups;
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
index 6f612af..dfd0d15 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
@@ -97,6 +97,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
private static final String COMPONENT_HOSTNAMES_PROPERTY = "hostnames";
private static final String CONFIGURATIONS_PROPERTY = "configurations";
private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations";
+ private static final String USER_CONTEXT_PROPERTY = "user-context";
private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties";
private File recommendationsDir;
@@ -205,6 +206,9 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
JsonNode changedConfigs = mapper.valueToTree(request.getChangedConfigurations());
root.put(CHANGED_CONFIGURATIONS_PROPERTY, changedConfigs);
+
+ JsonNode userContext = mapper.valueToTree(request.getUserContext());
+ root.put(USER_CONTEXT_PROPERTY, userContext);
}
private void populateConfigGroups(ObjectNode root,
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
index 0ad9126..5bb52c2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
@@ -58,6 +58,9 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
private static final String SERVICES_PROPERTY = "services";
private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed_configurations";
+ private static final String OPERATION_PROPERTY = "operation";
+ private static final String OPERATION_DETAILS_PROPERTY = "operation_details";
+
private static final String BLUEPRINT_HOST_GROUPS_PROPERTY = "recommendations/blueprint/host_groups";
private static final String BINDING_HOST_GROUPS_PROPERTY = "recommendations/blueprint_cluster_binding/host_groups";
@@ -75,6 +78,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
private static final String CONFIG_GROUPS_HOSTS_PROPERTY = "hosts";
protected static StackAdvisorHelper saHelper;
+ protected static final String USER_CONTEXT_OPERATION_PROPERTY = "user_context/operation";
+ protected static final String USER_CONTEXT_OPERATION_DETAILS_PROPERTY = "user_context/operation_details";
@Inject
public static void init(StackAdvisorHelper instance) {
@@ -109,6 +114,7 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
Map<String, Set<String>> componentHostsMap = calculateComponentHostsMap(hgComponentsMap,
hgHostsMap);
Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request);
+ Map<String, String> userContext = readUserContext(request);
List<ChangedConfigInfo> changedConfigurations =
requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES ?
@@ -122,9 +128,10 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
withComponentHostsMap(componentHostsMap).
withConfigurations(configurations).
withConfigGroups(configGroups).
- withChangedConfigurations(changedConfigurations).build();
+ withChangedConfigurations(changedConfigurations).
+ withUserContext(userContext).build();
} catch (Exception e) {
- LOG.warn("Error occured during preparation of stack advisor request", e);
+ LOG.warn("Error occurred during preparation of stack advisor request", e);
Response response = Response.status(Status.BAD_REQUEST)
.entity(String.format("Request body is not correct, error: %s", e.getMessage())).build();
// TODO: Hosts and services must not be empty
@@ -240,6 +247,26 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
return configGroups;
}
+ /**
+ * Parse the user contex for the call. Typical structure
+ * { "operation" : "createCluster" }
+ * { "operation" : "addService", "services" : "Atlas,Slider" }
+ * @param request
+ * @return
+ */
+ protected Map<String, String> readUserContext(Request request) {
+ HashMap<String, String> userContext = new HashMap<>();
+ if (null != getRequestProperty(request, USER_CONTEXT_OPERATION_PROPERTY)) {
+ userContext.put(OPERATION_PROPERTY,
+ (String) getRequestProperty(request, USER_CONTEXT_OPERATION_PROPERTY));
+ }
+ if (null != getRequestProperty(request, USER_CONTEXT_OPERATION_DETAILS_PROPERTY)) {
+ userContext.put(OPERATION_DETAILS_PROPERTY,
+ (String) getRequestProperty(request, USER_CONTEXT_OPERATION_DETAILS_PROPERTY));
+ }
+ return userContext;
+ }
+
protected static final String CONFIGURATIONS_PROPERTY_ID = "recommendations/blueprint/configurations/";
protected Map<String, Map<String, Map<String, String>>> calculateConfigurations(Request request) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/resources/properties.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json
index b7e0988..698b6c5 100644
--- a/ambari-server/src/main/resources/properties.json
+++ b/ambari-server/src/main/resources/properties.json
@@ -405,6 +405,9 @@
"hosts",
"services",
"changed_configurations",
+ "user_context",
+ "user_context/operation",
+ "user_context/operation_details",
"recommendations",
"recommendations/blueprint",
"recommendations/blueprint/configurations",
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/resources/scripts/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/stack_advisor.py b/ambari-server/src/main/resources/scripts/stack_advisor.py
index 56bd34a..66ab248 100755
--- a/ambari-server/src/main/resources/scripts/stack_advisor.py
+++ b/ambari-server/src/main/resources/scripts/stack_advisor.py
@@ -42,6 +42,10 @@ STACK_ADVISOR_DEFAULT_IMPL_CLASS = 'DefaultStackAdvisor'
STACK_ADVISOR_IMPL_PATH_TEMPLATE = os.path.join(SCRIPT_DIRECTORY, './../stacks/{0}/{1}/services/stack_advisor.py')
STACK_ADVISOR_IMPL_CLASS_TEMPLATE = '{0}{1}StackAdvisor'
+ADVISOR_CONTEXT = "advisor_context"
+CALL_TYPE = "call_type"
+
+
class StackAdvisorException(Exception):
pass
@@ -100,23 +104,23 @@ def main(argv=None):
hosts = stackAdvisor.filterHostMounts(hosts, services)
if action == RECOMMEND_COMPONENT_LAYOUT_ACTION:
- services['context'] = {'call_type': 'recommendComponentLayout'}
+ services[ADVISOR_CONTEXT] = {CALL_TYPE : 'recommendComponentLayout'}
result = stackAdvisor.recommendComponentLayout(services, hosts)
result_file = os.path.join(actionDir, "component-layout.json")
elif action == VALIDATE_COMPONENT_LAYOUT_ACTION:
- services['context'] = {'call_type': 'validateComponentLayout'}
+ services[ADVISOR_CONTEXT] = {CALL_TYPE : 'validateComponentLayout'}
result = stackAdvisor.validateComponentLayout(services, hosts)
result_file = os.path.join(actionDir, "component-layout-validation.json")
elif action == RECOMMEND_CONFIGURATIONS:
- services['context'] = {'call_type': 'recommendConfigurations'}
+ services[ADVISOR_CONTEXT] = {CALL_TYPE : 'recommendConfigurations'}
result = stackAdvisor.recommendConfigurations(services, hosts)
result_file = os.path.join(actionDir, "configurations.json")
elif action == RECOMMEND_CONFIGURATION_DEPENDENCIES:
- services['context'] = {'call_type': 'recommendConfigurationDependencies'}
+ services[ADVISOR_CONTEXT] = {CALL_TYPE : 'recommendConfigurationDependencies'}
result = stackAdvisor.recommendConfigurationDependencies(services, hosts)
result_file = os.path.join(actionDir, "configurations.json")
else: # action == VALIDATE_CONFIGURATIONS
- services['context'] = {'call_type': 'validateConfigurations'}
+ services[ADVISOR_CONTEXT] = {CALL_TYPE: 'validateConfigurations'}
result = stackAdvisor.validateConfigurations(services, hosts)
result_file = os.path.join(actionDir, "configurations-validation.json")
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 968a94f..628f183 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -1044,9 +1044,16 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
suggestedMinContainerRam = 1024 # new smaller value for YARN min container
callContext = getCallContext(services)
+ operation = getUserOperationContext(services, DefaultStackAdvisor.OPERATION)
+ if operation:
+ Logger.info("user operation context : " + str(operation))
+
if services: # its never None but some unit tests pass it as None
- if None != getOldValue(self, services, "yarn-site", "yarn.scheduler.minimum-allocation-mb") or \
- 'recommendConfigurations' != callContext:
+ # If min container value is changed (user is changing it)
+ # if its a validation call - just used what ever value is set
+ # If its not a cluster create or add yarn service (TBD)
+ if (getOldValue(self, services, "yarn-site", "yarn.scheduler.minimum-allocation-mb") or \
+ 'recommendConfigurations' != callContext) and operation != DefaultStackAdvisor.CLUSTER_CREATE_OPERATION:
'''yarn.scheduler.minimum-allocation-mb has changed - then pick this value up'''
if "yarn-site" in services["configurations"] and \
"yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"] and \
@@ -1964,9 +1971,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
def getCallContext(services):
if services:
- if 'context' in services:
- Logger.info("context : " + str (services['context']))
- return services['context']['call_type']
+ if DefaultStackAdvisor.ADVISOR_CONTEXT in services:
+ Logger.info("call type context : " + str(services[DefaultStackAdvisor.ADVISOR_CONTEXT]))
+ return services[DefaultStackAdvisor.ADVISOR_CONTEXT][DefaultStackAdvisor.CALL_TYPE]
return ""
@@ -1979,6 +1986,14 @@ def getOldValue(self, services, configType, propertyName):
return changedConfig["old_value"]
return None
+def getUserOperationContext(services, contextName):
+ if services:
+ if 'user-context' in services.keys():
+ userContext = services["user-context"]
+ if contextName in userContext:
+ return userContext[contextName]
+ return None
+
# Validation helper methods
def getSiteProperties(configurations, siteName):
siteConfig = configurations.get(siteName)
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 6b29bc3..2dce53a 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -300,6 +300,17 @@ class StackAdvisor(object):
class DefaultStackAdvisor(StackAdvisor):
+
+ CLUSTER_CREATE_OPERATION = "ClusterCreate"
+ ADD_SERVICE_OPERATION = "AddService"
+ EDIT_CONFIG_OPERATION = "EditConfig"
+ RECOMMEND_ATTRIBUTE_OPERATION = "RecommendAttribute"
+ OPERATION = "operation"
+ OPERATION_DETAILS = "operation_details"
+
+ ADVISOR_CONTEXT = "advisor_context"
+ CALL_TYPE = "call_type"
+
"""
Default stack advisor implementation.
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProviderTest.java
index e3b89b8..5a3c8af 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProviderTest.java
@@ -32,6 +32,8 @@ import java.util.Map;
import java.util.Set;
import static org.apache.ambari.server.controller.internal.StackAdvisorResourceProvider.CONFIGURATIONS_PROPERTY_ID;
+import static org.apache.ambari.server.controller.internal.StackAdvisorResourceProvider.USER_CONTEXT_OPERATION_PROPERTY;
+import static org.apache.ambari.server.controller.internal.StackAdvisorResourceProvider.USER_CONTEXT_OPERATION_DETAILS_PROPERTY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -76,6 +78,36 @@ public class StackAdvisorResourceProviderTest {
}
@Test
+ public void testReadUserContext() throws Exception {
+
+ Map<Resource.Type, String> keyPropertyIds = Collections.emptyMap();
+ Set<String> propertyIds = Collections.emptySet();
+ AmbariManagementController ambariManagementController = mock(AmbariManagementController.class);
+ RecommendationResourceProvider provider = new RecommendationResourceProvider(propertyIds,
+ keyPropertyIds, ambariManagementController);
+
+ Request request = mock(Request.class);
+ Set<Map<String, Object>> propertiesSet = new HashSet<Map<String, Object>>();
+ Map<String, Object> propertiesMap = new HashMap<String, Object>();
+ propertiesMap.put(CONFIGURATIONS_PROPERTY_ID + "site/properties/string_prop", "string");
+ List<Object> array = new ArrayList<Object>();
+ array.add("array1");
+ array.add("array2");
+ propertiesMap.put(USER_CONTEXT_OPERATION_PROPERTY, "op1");
+ propertiesMap.put(USER_CONTEXT_OPERATION_DETAILS_PROPERTY, "op_det");
+ propertiesSet.add(propertiesMap);
+
+ doReturn(propertiesSet).when(request).getProperties();
+
+ Map<String, String> userContext = provider.readUserContext(request);
+
+ assertNotNull(userContext);
+ assertEquals(2, userContext.size());
+ assertEquals("op1", userContext.get("operation"));
+ assertEquals("op_det", userContext.get("operation_details"));
+ }
+
+ @Test
public void testCalculateConfigurationsWithNullPropertyValues() throws Exception {
Map<Resource.Type, String> keyPropertyIds = Collections.emptyMap();
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ef624fe/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index f39703c..c2c2c2d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -526,7 +526,7 @@ class TestHDP206StackAdvisor(TestCase):
}
},
"changed-configurations": [],
- "context": {'call_type': 'validateConfigurations'}
+ "advisor_context": {'call_type': 'validateConfigurations'}
}
result = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, services)
self.assertEquals(result, expected)
@@ -571,7 +571,7 @@ class TestHDP206StackAdvisor(TestCase):
}
},
"changed-configurations": [],
- "context": {'call_type': 'validateConfigurations'}
+ "advisor_context": {'call_type': 'validateConfigurations'}
}
expected_2048 = {
@@ -636,7 +636,7 @@ class TestHDP206StackAdvisor(TestCase):
}
},
"changed-configurations": [],
- "context": {'call_type': 'recommendConfigurationDependencies'}
+ "advisor_context": {'call_type': 'recommendConfigurationDependencies'}
}
expected_2048 = {
@@ -701,7 +701,7 @@ class TestHDP206StackAdvisor(TestCase):
}
},
"changed-configurations": [],
- "context": {'call_type': 'recommendConfigurationDependencies'}
+ "advisor_context": {'call_type': 'recommendConfigurationDependencies'}
}
expected_4096 = {