You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2017/11/20 18:59:00 UTC
ambari git commit: AMBARI-22484. Stack advisor should disallow lzo
enable without accepting license agreement. (Myroslav Papirkovskyy via
swagle)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-22457 1fce5650b -> 9d55c0f50
AMBARI-22484. Stack advisor should disallow lzo enable without accepting license agreement. (Myroslav Papirkovskyy via swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9d55c0f5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9d55c0f5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9d55c0f5
Branch: refs/heads/branch-feature-AMBARI-22457
Commit: 9d55c0f5034c2765e4428d9b3ad56c455be54bfc
Parents: 1fce565
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Mon Nov 20 10:58:47 2017 -0800
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Mon Nov 20 10:58:47 2017 -0800
----------------------------------------------------------------------
.../stackadvisor/StackAdvisorRequest.java | 11 ++++++++
.../commands/StackAdvisorCommand.java | 2 ++
.../ambari/server/controller/AmbariServer.java | 3 ++-
.../internal/StackAdvisorResourceProvider.java | 9 +++++--
.../GPLLicenseNotAcceptedException.java | 28 ++++++++++++++++++++
.../stacks/HDP/2.0.6/services/stack_advisor.py | 23 +++++++++++++++-
.../ValidationResourceProviderTest.java | 4 ++-
.../stacks/2.0.6/common/test_stack_advisor.py | 25 +++++++++++++++++
8 files changed, 100 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
index 64180e5..bc1e079 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
@@ -48,6 +48,7 @@ public class StackAdvisorRequest {
private List<ChangedConfigInfo> changedConfigurations = new LinkedList<ChangedConfigInfo>();
private Set<RecommendationResponse.ConfigGroup> configGroups;
private Map<String, String> userContext = new HashMap<String, String>();
+ private Boolean gplLicenseAccepted;
public String getStackName() {
return stackName;
@@ -117,6 +118,10 @@ public class StackAdvisorRequest {
this.configGroups = configGroups;
}
+ public Boolean getGplLicenseAccepted() {
+ return gplLicenseAccepted;
+ }
+
private StackAdvisorRequest(String stackName, String stackVersion) {
this.stackName = stackName;
this.stackVersion = stackVersion;
@@ -189,6 +194,12 @@ public class StackAdvisorRequest {
return this;
}
+ public StackAdvisorRequestBuilder withGPLLicenseAccepted(
+ Boolean gplLicenseAccepted) {
+ this.instance.gplLicenseAccepted = gplLicenseAccepted;
+ return this;
+ }
+
public StackAdvisorRequest build() {
return this.instance;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
index 5440462..48924f8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
@@ -94,6 +94,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
private static final String CONFIGURATIONS_PROPERTY = "configurations";
private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations";
private static final String USER_CONTEXT_PROPERTY = "user-context";
+ private static final String GPL_LICENSE_ACCEPTED = "gpl-license-accepted";
private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties";
private File recommendationsDir;
@@ -205,6 +206,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
JsonNode userContext = mapper.valueToTree(request.getUserContext());
root.put(USER_CONTEXT_PROPERTY, userContext);
+ root.put(GPL_LICENSE_ACCEPTED, request.getGplLicenseAccepted());
}
private void populateConfigGroups(ObjectNode root,
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index b2a68d6..154d4e7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -904,7 +904,8 @@ public class AmbariServer {
PersistKeyValueService.init(injector.getInstance(PersistKeyValueImpl.class));
KeyService.init(injector.getInstance(PersistKeyValueImpl.class));
BootStrapResource.init(injector.getInstance(BootStrapImpl.class));
- StackAdvisorResourceProvider.init(injector.getInstance(StackAdvisorHelper.class));
+ StackAdvisorResourceProvider.init(injector.getInstance(StackAdvisorHelper.class),
+ injector.getInstance(Configuration.class));
StageUtils.setGson(injector.getInstance(Gson.class));
StageUtils.setTopologyManager(injector.getInstance(TopologyManager.class));
StageUtils.setConfiguration(injector.getInstance(Configuration.class));
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
index dd32c6b..c2993db 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
@@ -36,6 +36,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestBuilder;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType;
import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
+import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.spi.Request;
import org.apache.ambari.server.controller.spi.Resource.Type;
@@ -79,12 +80,14 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
private static final String CONFIG_GROUPS_HOSTS_PROPERTY = "hosts";
protected static StackAdvisorHelper saHelper;
+ protected static Configuration configuration;
protected static final String USER_CONTEXT_OPERATION_PROPERTY = "user_context/operation";
protected static final String USER_CONTEXT_OPERATION_DETAILS_PROPERTY = "user_context/operation_details";
@Inject
- public static void init(StackAdvisorHelper instance) {
+ public static void init(StackAdvisorHelper instance, Configuration serverConfig) {
saHelper = instance;
+ configuration = serverConfig;
}
protected StackAdvisorResourceProvider(Set<String> propertyIds, Map<Type, String> keyPropertyIds,
@@ -130,6 +133,7 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
hgHostsMap);
Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request);
Map<String, String> userContext = readUserContext(request);
+ Boolean gplLicenseAccepted = configuration.getGplLicenseAccepted();
List<ChangedConfigInfo> changedConfigurations =
requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES ?
@@ -144,7 +148,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
withConfigurations(configurations).
withConfigGroups(configGroups).
withChangedConfigurations(changedConfigurations).
- withUserContext(userContext).build();
+ withUserContext(userContext).
+ withGPLLicenseAccepted(gplLicenseAccepted).build();
} catch (Exception e) {
LOG.warn("Error occurred during preparation of stack advisor request", e);
Response response = Response.status(Status.BAD_REQUEST)
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
new file mode 100644
index 0000000..b444d01
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+/**
+ * Indicates an not permitted LZO usage.
+ */
+public class GPLLicenseNotAcceptedException extends Exception {
+ public GPLLicenseNotAcceptedException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index c36a41f..6d80f1a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -516,6 +516,22 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
# recommendations for "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" properties in core-site
self.recommendHadoopProxyUsers(configurations, services, hosts)
+ def getLZOSupportValidationItems(self, properties, services):
+ services_list = self.get_services_list(services)
+
+ if "HDFS" in services_list:
+ lzo_allowed = services["gpl-license-accepted"]
+ property_name = "io.compression.codec.lzo.class"
+ if property_name in properties:
+ property_value = properties.get(property_name)
+ if not lzo_allowed and "com.hadoop.compression.lzo.LzoCodec" in property_value:
+ return [{"config-name": property_name, "item": self.getErrorItem(
+ "Your Ambari Server has not been configured to download LZO and install it. "
+ "LZO is GPL software and requires you to accept a license prior to use. "
+ "Please refer to this documentation to configure Ambari before proceeding.")}]
+
+ return []
+
def recommendHbaseConfigurations(self, configurations, clusterData, services, hosts):
# recommendations for HBase env config
@@ -1173,7 +1189,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
def getServiceConfigurationValidators(self):
return {
"HDFS": { "hdfs-site": self.validateHDFSConfigurations,
- "hadoop-env": self.validateHDFSConfigurationsEnv},
+ "hadoop-env": self.validateHDFSConfigurationsEnv,
+ "core-site": self.validateHDFSConfigurationsCoreSite},
"MAPREDUCE2": {"mapred-site": self.validateMapReduce2Configurations},
"YARN": {"yarn-site": self.validateYARNConfigurations,
"yarn-env": self.validateYARNEnvConfigurations},
@@ -1783,6 +1800,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
{"config-name": 'namenode_opt_maxnewsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_opt_maxnewsize')}]
return self.toConfigurationValidationProblems(validationItems, "hadoop-env")
+ def validateHDFSConfigurationsCoreSite(self, properties, recommendedDefaults, configurations, services, hosts):
+ return self.toConfigurationValidationProblems(self.getLZOSupportValidationItems(properties, services),
+ "core-site")
+
def validatorOneDataDirPerPartition(self, properties, propertyName, services, hosts, clusterEnv):
if not propertyName in properties:
return self.getErrorItem("Value should be set")
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
index cdbf2d7..e245e7f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
@@ -21,6 +21,7 @@ package org.apache.ambari.server.controller.internal;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
+import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.spi.Request;
import org.apache.ambari.server.controller.spi.RequestStatus;
@@ -49,13 +50,14 @@ public class ValidationResourceProviderTest {
doReturn(stackAdvisorRequest).when(provider).prepareStackAdvisorRequest(request);
StackAdvisorHelper saHelper = mock(StackAdvisorHelper.class);
+ Configuration configuration = mock(Configuration.class);
ValidationResponse response = mock(ValidationResponse.class);
Version version = mock(Version.class);
doReturn(3).when(response).getId();
doReturn(version).when(response).getVersion();
doReturn(response).when(saHelper).validate(any(StackAdvisorRequest.class));
- ValidationResourceProvider.init(saHelper);
+ ValidationResourceProvider.init(saHelper, configuration);
RequestStatus status = provider.createResources(request);
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d55c0f5/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 9e00704..e011847 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -2356,6 +2356,31 @@ class TestHDP206StackAdvisor(TestCase):
res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
self.assertEquals(res, res_expected)
+ def test_validateHDFSConfigurationsCoreSite(self):
+
+ configurations = {}
+ services = {"gpl-license-accepted": True, "services": [{"StackServices": {"service_name": "HDFS"}}]}
+
+ # 1) ok: gpl is allowed
+ properties = {'io.compression.codec.lzo.class': 'com.hadoop.compression.lzo.LzoCodec'}
+ res_expected = []
+
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, '')
+ self.assertEquals(res, res_expected)
+
+ # 2) fail: gpl is not allowed
+ services["gpl-license-accepted"] = False
+ res_expected = [{'config-type': 'core-site',
+ 'message': 'Your Ambari Server has not been configured to download LZO and install it. '
+ 'LZO is GPL software and requires you to accept a license prior to use. '
+ 'Please refer to this documentation to configure Ambari before proceeding.',
+ 'type': 'configuration',
+ 'config-name': 'io.compression.codec.lzo.class',
+ 'level': 'ERROR'}]
+
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, '')
+ self.assertEquals(res, res_expected)
+
def test_validateOneDataDirPerPartition(self):
recommendedDefaults = {
'dfs.datanode.du.reserved': '1024'