You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mp...@apache.org on 2017/11/29 16:30:31 UTC
[3/5] ambari git commit: AMBARI-22484. Stack advisor should disallow
lzo enable without accepting license agreement. (mpapirkovskyy)
AMBARI-22484. Stack advisor should disallow lzo enable without accepting license agreement. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/41853a10
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/41853a10
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/41853a10
Branch: refs/heads/trunk
Commit: 41853a1041a848b69384592c49f8f5b539a5c4f9
Parents: 7750398
Author: Myroslav Papirkovskyi <mp...@hortonworks.com>
Authored: Wed Nov 29 16:48:11 2017 +0200
Committer: Myroslav Papirkovskyi <mp...@hortonworks.com>
Committed: Wed Nov 29 18:30:08 2017 +0200
----------------------------------------------------------------------
.../stackadvisor/StackAdvisorRequest.java | 11 +++++
.../commands/StackAdvisorCommand.java | 2 +
.../ambari/server/controller/AmbariServer.java | 3 +-
.../internal/StackAdvisorResourceProvider.java | 9 +++-
.../GPLLicenseNotAcceptedException.java | 28 +++++++++++
.../stacks/HDP/2.0.6/services/stack_advisor.py | 17 +++++++
.../ValidationResourceProviderTest.java | 4 +-
.../stacks/2.0.6/common/test_stack_advisor.py | 51 ++++++++++++++++++--
8 files changed, 116 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
index cd26c56..62b8d15 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
@@ -51,6 +51,7 @@ public class StackAdvisorRequest {
private Set<RecommendationResponse.ConfigGroup> configGroups;
private Map<String, String> userContext = new HashMap<>();
private Map<String, Object> ldapConfig = new HashMap<>();
+ private Boolean gplLicenseAccepted;
public String getStackName() {
return stackName;
@@ -122,6 +123,10 @@ public class StackAdvisorRequest {
this.configGroups = configGroups;
}
+ public Boolean getGplLicenseAccepted() {
+ return gplLicenseAccepted;
+ }
+
private StackAdvisorRequest(String stackName, String stackVersion) {
this.stackName = stackName;
this.stackVersion = stackVersion;
@@ -194,6 +199,12 @@ public class StackAdvisorRequest {
return this;
}
+ public StackAdvisorRequestBuilder withGPLLicenseAccepted(
+ Boolean gplLicenseAccepted) {
+ this.instance.gplLicenseAccepted = gplLicenseAccepted;
+ return this;
+ }
+
public StackAdvisorRequestBuilder withLdapConfig(Map<String, Object> ldapConfig) {
Preconditions.checkNotNull(ldapConfig);
this.instance.ldapConfig = ldapConfig;
http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
index 1b89c4f..4ec15ef 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
@@ -107,6 +107,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
private static final String CONFIGURATIONS_PROPERTY = "configurations";
private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations";
private static final String USER_CONTEXT_PROPERTY = "user-context";
+ private static final String GPL_LICENSE_ACCEPTED = "gpl-license-accepted";
private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties";
private File recommendationsDir;
@@ -269,6 +270,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend
JsonNode userContext = mapper.valueToTree(request.getUserContext());
root.put(USER_CONTEXT_PROPERTY, userContext);
+ root.put(GPL_LICENSE_ACCEPTED, request.getGplLicenseAccepted());
}
private void populateConfigGroups(ObjectNode root,
http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index 6ceed4a..863313b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -908,7 +908,8 @@ public class AmbariServer {
PersistKeyValueService.init(injector.getInstance(PersistKeyValueImpl.class));
KeyService.init(injector.getInstance(PersistKeyValueImpl.class));
BootStrapResource.init(injector.getInstance(BootStrapImpl.class));
- StackAdvisorResourceProvider.init(injector.getInstance(StackAdvisorHelper.class));
+ StackAdvisorResourceProvider.init(injector.getInstance(StackAdvisorHelper.class),
+ injector.getInstance(Configuration.class));
StageUtils.setGson(injector.getInstance(Gson.class));
StageUtils.setTopologyManager(injector.getInstance(TopologyManager.class));
StageUtils.setConfiguration(injector.getInstance(Configuration.class));
http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
index 1ea664c..67c177e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
@@ -36,6 +36,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestBuilder;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType;
import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
+import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.spi.Request;
import org.apache.ambari.server.controller.spi.Resource.Type;
@@ -82,12 +83,14 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
private static final String CONFIG_GROUPS_HOSTS_PROPERTY = "hosts";
protected static StackAdvisorHelper saHelper;
+ protected static Configuration configuration;
protected static final String USER_CONTEXT_OPERATION_PROPERTY = "user_context/operation";
protected static final String USER_CONTEXT_OPERATION_DETAILS_PROPERTY = "user_context/operation_details";
@Inject
- public static void init(StackAdvisorHelper instance) {
+ public static void init(StackAdvisorHelper instance, Configuration serverConfig) {
saHelper = instance;
+ configuration = serverConfig;
}
protected StackAdvisorResourceProvider(Set<String> propertyIds, Map<Type, String> keyPropertyIds,
@@ -133,6 +136,7 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
hgHostsMap);
Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request);
Map<String, String> userContext = readUserContext(request);
+ Boolean gplLicenseAccepted = configuration.getGplLicenseAccepted();
List<ChangedConfigInfo> changedConfigurations =
requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES ?
@@ -147,7 +151,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
withConfigurations(configurations).
withConfigGroups(configGroups).
withChangedConfigurations(changedConfigurations).
- withUserContext(userContext).build();
+ withUserContext(userContext).
+ withGPLLicenseAccepted(gplLicenseAccepted).build();
} catch (Exception e) {
LOG.warn("Error occurred during preparation of stack advisor request", e);
Response response = Response.status(Status.BAD_REQUEST)
http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
new file mode 100644
index 0000000..b444d01
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+/**
+ * Indicates an not permitted LZO usage.
+ */
+public class GPLLicenseNotAcceptedException extends Exception {
+ public GPLLicenseNotAcceptedException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index a194332..bfa2f5a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -351,6 +351,22 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
# recommendations for "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" properties in core-site
self.recommendHadoopProxyUsers(configurations, services, hosts)
+ def getLZOSupportValidationItems(self, properties, services):
+ services_list = self.get_services_list(services)
+
+ if "HDFS" in services_list:
+ lzo_allowed = services["gpl-license-accepted"]
+ property_name = "io.compression.codec.lzo.class"
+ if property_name in properties:
+ property_value = properties.get(property_name)
+ if not lzo_allowed and "com.hadoop.compression.lzo.LzoCodec" in property_value:
+ return [{"config-name": property_name, "item": self.getErrorItem(
+ "Your Ambari Server has not been configured to download LZO and install it. "
+ "LZO is GPL software and requires you to accept a license prior to use. "
+ "Please refer to this documentation to configure Ambari before proceeding.")}]
+
+ return []
+
def recommendHbaseConfigurations(self, configurations, clusterData, services, hosts):
# recommendations for HBase env config
@@ -726,6 +742,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
validationItems = []
validationItems.extend(self.getHadoopProxyUsersValidationItems(properties, services, hosts, configurations))
validationItems.extend(self.getAmbariProxyUsersForHDFSValidationItems(properties, services))
+ validationItems.extend(self.getLZOSupportValidationItems(properties, services))
return self.toConfigurationValidationProblems(validationItems, "core-site")
def validatorOneDataDirPerPartition(self, properties, propertyName, services, hosts, clusterEnv):
http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
index c639d1f..42d4770 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
@@ -33,6 +33,7 @@ import java.util.Set;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
+import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.spi.Request;
import org.apache.ambari.server.controller.spi.RequestStatus;
@@ -53,13 +54,14 @@ public class ValidationResourceProviderTest {
doReturn(stackAdvisorRequest).when(provider).prepareStackAdvisorRequest(request);
StackAdvisorHelper saHelper = mock(StackAdvisorHelper.class);
+ Configuration configuration = mock(Configuration.class);
ValidationResponse response = mock(ValidationResponse.class);
Version version = mock(Version.class);
doReturn(3).when(response).getId();
doReturn(version).when(response).getVersion();
doReturn(response).when(saHelper).validate(any(StackAdvisorRequest.class));
- ValidationResourceProvider.init(saHelper);
+ ValidationResourceProvider.init(saHelper, configuration);
RequestStatus status = provider.createResources(request);
http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 38d6ecd..6c774af 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -2521,7 +2521,8 @@ class TestHDP206StackAdvisor(TestCase):
'hadoop.proxyuser.hdfs-user.hosts': '*',
'hadoop.proxyuser.hdfs-user.groups': '*',
'hadoop.proxyuser.yarn-user.hosts': 'host1,host2',
- 'hadoop.proxyuser.yarn-user.groups': '*'}
+ 'hadoop.proxyuser.yarn-user.groups': '*',
+ 'io.compression.codec.lzo.class': 'com.hadoop.compression.lzo.LzoCodec'}
services = {
'services': [
{ 'StackServices': {'service_name': 'HDFS'}},
@@ -2537,7 +2538,8 @@ class TestHDP206StackAdvisor(TestCase):
}
],
'ambari-server-properties': {'ambari-server.user': 'ambari-user'},
- 'configurations': configurations
+ 'configurations': configurations,
+ "gpl-license-accepted": True
}
hosts = {
'items' : [
@@ -2551,7 +2553,21 @@ class TestHDP206StackAdvisor(TestCase):
res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts)
self.assertEquals(res, res_expected)
- # 2) fail: test filter function: two RESOURCE_MANAGERs, hadoop.proxyuser.yarn-user.hosts is expected to be set
+ # 2) fail: gpl is not allowed
+ services["gpl-license-accepted"] = False
+ res_expected = [{'config-type': 'core-site',
+ 'message': 'Your Ambari Server has not been configured to download LZO and install it. '
+ 'LZO is GPL software and requires you to accept a license prior to use. '
+ 'Please refer to this documentation to configure Ambari before proceeding.',
+ 'type': 'configuration',
+ 'config-name': 'io.compression.codec.lzo.class',
+ 'level': 'ERROR'}]
+
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, hosts)
+ self.assertEquals(res, res_expected)
+ services["gpl-license-accepted"] = True
+
+ # 3) fail: test filter function: two RESOURCE_MANAGERs, hadoop.proxyuser.yarn-user.hosts is expected to be set
del properties['hadoop.proxyuser.yarn-user.hosts']
res_expected = [{'config-name': 'hadoop.proxyuser.yarn-user.hosts',
'config-type': 'core-site',
@@ -2561,13 +2577,13 @@ class TestHDP206StackAdvisor(TestCase):
res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts)
self.assertEquals(res, res_expected)
- # 3) ok: test filter function: only one RESOURCE_MANAGER
+ # 4) ok: test filter function: only one RESOURCE_MANAGER
services['services'][1]['components'][0]['StackServiceComponents']['hostnames'] = ["host1"]
res_expected = []
res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts)
self.assertEquals(res, res_expected)
- # 4) fail: some proxyusers are empty or absent:
+ # 5) fail: some proxyusers are empty or absent:
del properties['hadoop.proxyuser.ambari-user.hosts']
properties['hadoop.proxyuser.hdfs-user.groups'] = ''
res_expected = [{'config-name': 'hadoop.proxyuser.hdfs-user.groups',
@@ -2627,6 +2643,31 @@ class TestHDP206StackAdvisor(TestCase):
res = self.stackAdvisor.getHadoopProxyUsers(services, hosts, configurations)
self.assertEquals(res, res_expected)
+ # def test_validateHDFSConfigurationsCoreSite(self):
+ #
+ # configurations = {}
+ # services = {"gpl-license-accepted": True, "services": [{"StackServices": {"service_name": "HDFS"}}], 'ambari-server-properties': {'ambari-server.user': 'ambari-user'}}
+ #
+ # # 1) ok: gpl is allowed
+ # properties = {'io.compression.codec.lzo.class': 'com.hadoop.compression.lzo.LzoCodec'}
+ # res_expected = []
+ #
+ # res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, '')
+ # self.assertEquals(res, res_expected)
+ #
+ # # 2) fail: gpl is not allowed
+ # services["gpl-license-accepted"] = False
+ # res_expected = [{'config-type': 'core-site',
+ # 'message': 'Your Ambari Server has not been configured to download LZO and install it. '
+ # 'LZO is GPL software and requires you to accept a license prior to use. '
+ # 'Please refer to this documentation to configure Ambari before proceeding.',
+ # 'type': 'configuration',
+ # 'config-name': 'io.compression.codec.lzo.class',
+ # 'level': 'ERROR'}]
+ #
+ # res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations, services, '')
+ # self.assertEquals(res, res_expected)
+
def test_validateOneDataDirPerPartition(self):
recommendedDefaults = {
'dfs.datanode.du.reserved': '1024'