You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mp...@apache.org on 2017/11/29 16:30:33 UTC
[5/5] ambari git commit: AMBARI-22480. Validate blueprint does not
allow lzo enable without setup with license agreement. (mpapirkovskyy)
AMBARI-22480. Validate blueprint does not allow lzo enable without setup with license agreement. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9bbc0ef7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9bbc0ef7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9bbc0ef7
Branch: refs/heads/trunk
Commit: 9bbc0ef720bc91ff39701955984bb9635d811c59
Parents: 41853a1
Author: Myroslav Papirkovskyi <mp...@hortonworks.com>
Authored: Wed Nov 29 16:55:34 2017 +0200
Committer: Myroslav Papirkovskyi <mp...@hortonworks.com>
Committed: Wed Nov 29 18:30:08 2017 +0200
----------------------------------------------------------------------
.../internal/BlueprintResourceProvider.java | 3 +-
.../ambari/server/topology/Blueprint.java | 2 +-
.../ambari/server/topology/BlueprintImpl.java | 2 +-
.../server/topology/BlueprintValidator.java | 2 +-
.../server/topology/BlueprintValidatorImpl.java | 21 +++++-
.../server/topology/BlueprintImplTest.java | 69 ++++++++++++++++++--
.../topology/BlueprintValidatorImplTest.java | 10 ++-
7 files changed, 95 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
index 67f5448..8f4d62e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
@@ -56,6 +56,7 @@ import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.topology.Blueprint;
import org.apache.ambari.server.topology.BlueprintFactory;
+import org.apache.ambari.server.topology.GPLLicenseNotAcceptedException;
import org.apache.ambari.server.topology.InvalidTopologyException;
import org.apache.ambari.server.topology.SecurityConfiguration;
import org.apache.ambari.server.topology.SecurityConfigurationFactory;
@@ -519,7 +520,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
try {
blueprint.validateRequiredProperties();
- } catch (InvalidTopologyException e) {
+ } catch (InvalidTopologyException | GPLLicenseNotAcceptedException e) {
throw new IllegalArgumentException("Blueprint configuration validation failed: " + e.getMessage(), e);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java
index 779a02d..6ed38f8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java
@@ -152,7 +152,7 @@ public interface Blueprint {
*
* @throws InvalidTopologyException if the blueprint doesn't contain all required properties
*/
- void validateRequiredProperties() throws InvalidTopologyException;
+ void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException;
/**
*
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
index 8c83ed6..6801e33 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
@@ -353,7 +353,7 @@ public class BlueprintImpl implements Blueprint {
* @throws InvalidTopologyException if the blueprint configuration is invalid
*/
@Override
- public void validateRequiredProperties() throws InvalidTopologyException {
+ public void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException {
validator.validateRequiredProperties();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java
index be194df..156fe8c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java
@@ -37,5 +37,5 @@ public interface BlueprintValidator {
*
* @throws InvalidTopologyException if required properties are not set in blueprint
*/
- void validateRequiredProperties() throws InvalidTopologyException;
+ void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
index 1a43b85..87b5936 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -26,6 +26,7 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Set;
+import org.apache.ambari.server.StaticallyInject;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.state.AutoDeployInfo;
import org.apache.ambari.server.state.DependencyConditionInfo;
@@ -35,15 +36,24 @@ import org.apache.ambari.server.utils.VersionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.inject.Inject;
+
/**
* Default blueprint validator.
*/
+@StaticallyInject
public class BlueprintValidatorImpl implements BlueprintValidator {
private static final Logger LOGGER = LoggerFactory.getLogger(BlueprintValidatorImpl.class);
private final Blueprint blueprint;
private final Stack stack;
+ public static final String LZO_CODEC_CLASS_PROPERTY_NAME = "io.compression.codec.lzo.class";
+ public static final String LZO_CODEC_CLASS = "com.hadoop.compression.lzo.LzoCodec";
+
+ @Inject
+ private static org.apache.ambari.server.configuration.Configuration configuration;
+
public BlueprintValidatorImpl(Blueprint blueprint) {
this.blueprint = blueprint;
this.stack = blueprint.getStack();
@@ -84,13 +94,17 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
}
@Override
- public void validateRequiredProperties() throws InvalidTopologyException {
+ public void validateRequiredProperties() throws InvalidTopologyException, GPLLicenseNotAcceptedException {
// we don't want to include default stack properties so we can't just use hostGroup full properties
Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties();
// we need to have real passwords, not references
if (clusterConfigurations != null) {
+
+ // need to reject blueprints that have LZO enabled if the Ambari Server hasn't been configured for it
+ boolean gplEnabled = configuration.getGplLicenseAccepted();
+
StringBuilder errorMessage = new StringBuilder();
boolean containsSecretReferences = false;
for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet()) {
@@ -100,6 +114,11 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
String propertyName = propertyEntry.getKey();
String propertyValue = propertyEntry.getValue();
if (propertyValue != null) {
+ if (!gplEnabled && configType.equals("core-site") && propertyName.equals(LZO_CODEC_CLASS_PROPERTY_NAME)
+ && propertyValue.contains(LZO_CODEC_CLASS)) {
+ throw new GPLLicenseNotAcceptedException("Your Ambari server has not been configured to download LZO GPL software. " +
+ "Please refer to documentation to configure Ambari before proceeding.");
+ }
if (SecretReference.isSecret(propertyValue)) {
errorMessage.append(" Config:" + configType + " Property:" + propertyName + "\n");
containsSecretReferences = true;
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
index 6ac74a3..6d3179e 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
@@ -21,11 +21,13 @@ package org.apache.ambari.server.topology;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.mock;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -106,13 +108,14 @@ public class BlueprintImplTest {
@Test
public void testValidateConfigurations__basic_positive() throws Exception {
+ org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(true);
expect(group1.getCardinality()).andReturn("1").atLeastOnce();
expect(group1.getComponents()).andReturn(Arrays.asList(new Component("c1"), new Component("c2"))).atLeastOnce();
expect(group2.getCardinality()).andReturn("1").atLeastOnce();
expect(group2.getComponents()).andReturn(Arrays.asList(new Component("c1"), new Component("c3"))).atLeastOnce();
expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce();
- replay(stack, group1, group2);
+ replay(stack, group1, group2, serverConfig);
Map<String, String> category2Props = new HashMap<>();
properties.put("category2", category2Props);
@@ -123,7 +126,7 @@ public class BlueprintImplTest {
blueprint.validateRequiredProperties();
BlueprintEntity entity = blueprint.toEntity();
- verify(stack, group1, group2);
+ verify(stack, group1, group2, serverConfig);
assertTrue(entity.getSecurityType() == SecurityType.KERBEROS);
assertTrue(entity.getSecurityDescriptorReference().equals("testRef"));
}
@@ -134,6 +137,8 @@ public class BlueprintImplTest {
Map<String, String> group2Category2Props = new HashMap<>();
group2Props.put("category2", group2Category2Props);
group2Category2Props.put("prop2", "val");
+
+ org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(true);
// set config for group2 which contains a required property
Configuration group2Configuration = new Configuration(group2Props, EMPTY_ATTRIBUTES, configuration);
expect(group2.getConfiguration()).andReturn(group2Configuration).atLeastOnce();
@@ -155,11 +160,11 @@ public class BlueprintImplTest {
properties.put("hadoop-env", hadoopProps);
hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP:group1%");
hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP:group2%");
- replay(stack, group1, group2);
+ replay(stack, group1, group2, serverConfig);
Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null);
blueprint.validateRequiredProperties();
BlueprintEntity entity = blueprint.toEntity();
- verify(stack, group1, group2);
+ verify(stack, group1, group2, serverConfig);
assertTrue(entity.getSecurityType() == SecurityType.NONE);
assertTrue(entity.getSecurityDescriptorReference() == null);
}
@@ -277,17 +282,55 @@ public class BlueprintImplTest {
verify(stack, group1, group2);
}
@Test(expected = InvalidTopologyException.class)
- public void testValidateConfigurations__secretReference() throws InvalidTopologyException {
+ public void testValidateConfigurations__secretReference() throws InvalidTopologyException,
+ GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException {
Map<String, Map<String, String>> group2Props = new HashMap<>();
Map<String, String> group2Category2Props = new HashMap<>();
+
+ org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(true);
group2Props.put("category2", group2Category2Props);
group2Category2Props.put("prop2", "val");
hdfsProps.put("secret", "SECRET:hdfs-site:1:test");
- replay(stack, group1, group2);
+ replay(stack, group1, group2, serverConfig);
Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null);
blueprint.validateRequiredProperties();
- verify(stack, group1, group2);
+ verify(stack, group1, group2, serverConfig);
+ }
+
+ @Test(expected = GPLLicenseNotAcceptedException.class)
+ public void testValidateConfigurations__gplIsNotAllowed() throws InvalidTopologyException,
+ GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException {
+ Map<String, Map<String, String>> lzoProperties = new HashMap<>();
+ lzoProperties.put("core-site", new HashMap<String, String>(){{
+ put(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS);
+ }});
+ Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION);
+
+ org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(false);
+ replay(stack, group1, group2, serverConfig);
+
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null);
+ blueprint.validateRequiredProperties();
+ verify(stack, group1, group2, serverConfig);
+ }
+
+ @Test
+ public void testValidateConfigurations__gplISAllowed() throws InvalidTopologyException,
+ GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException {
+ Map<String, Map<String, String>> lzoProperties = new HashMap<>();
+ lzoProperties.put("core-site", new HashMap<String, String>(){{
+ put(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS);
+ }});
+ Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION);
+
+ org.apache.ambari.server.configuration.Configuration serverConfig = setupConfigurationWithGPLLicense(true);
+ expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce();
+ replay(stack, group1, group2, serverConfig);
+
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null);
+ blueprint.validateRequiredProperties();
+ verify(stack, group1, group2, serverConfig);
}
@Test
@@ -314,6 +357,18 @@ public class BlueprintImplTest {
verify(stack, setting);
}
+ public static org.apache.ambari.server.configuration.Configuration setupConfigurationWithGPLLicense(boolean isGPLAllowed)
+ throws NoSuchFieldException, IllegalAccessException {
+ org.apache.ambari.server.configuration.Configuration serverConfig =
+ mock(org.apache.ambari.server.configuration.Configuration.class);
+ expect(serverConfig.getGplLicenseAccepted()).andReturn(isGPLAllowed).atLeastOnce();
+
+ Field field = BlueprintValidatorImpl.class.getDeclaredField("configuration");
+ field.setAccessible(true);
+ field.set(null, serverConfig);
+ return serverConfig;
+ }
+
//todo: ensure coverage for these existing tests
// private void validateEntity(BlueprintEntity entity, boolean containsConfig) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9bbc0ef7/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
index a706428..75a9d6b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
@@ -228,6 +228,9 @@ public class BlueprintValidatorImplTest {
services.addAll(Arrays.asList("HIVE"));
+ org.apache.ambari.server.configuration.Configuration serverConfig =
+ BlueprintImplTest.setupConfigurationWithGPLLicense(true);
+
Configuration config = new Configuration(new HashMap<>(), new HashMap<>());
expect(group1.getConfiguration()).andReturn(config).anyTimes();
@@ -237,7 +240,7 @@ public class BlueprintValidatorImplTest {
expect(blueprint.getHostGroupsForComponent("HIVE_METASTORE")).andReturn(Collections.singleton(group1)).anyTimes();
- replay(blueprint, stack, group1, group2, dependency1);
+ replay(blueprint, stack, group1, group2, dependency1, serverConfig);
BlueprintValidator validator = new BlueprintValidatorImpl(blueprint);
validator.validateRequiredProperties();
}
@@ -252,6 +255,9 @@ public class BlueprintValidatorImplTest {
services.addAll(Arrays.asList("OOZIE"));
+ org.apache.ambari.server.configuration.Configuration serverConfig =
+ BlueprintImplTest.setupConfigurationWithGPLLicense(true);
+
Configuration config = new Configuration(new HashMap<>(), new HashMap<>());
expect(group1.getConfiguration()).andReturn(config).anyTimes();
@@ -261,7 +267,7 @@ public class BlueprintValidatorImplTest {
expect(blueprint.getHostGroupsForComponent("OOZIE_SERVER")).andReturn(Collections.singleton(group1)).anyTimes();
- replay(blueprint, stack, group1, group2, dependency1);
+ replay(blueprint, stack, group1, group2, dependency1, serverConfig);
BlueprintValidator validator = new BlueprintValidatorImpl(blueprint);
validator.validateRequiredProperties();
}