You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by lp...@apache.org on 2017/03/30 15:51:04 UTC
[4/4] ambari git commit: AMBARI-20594 Added hive service validator
impelemtation to signal invalid blueprint / cluster template
AMBARI-20594 Added hive service validator impelemtation to signal invalid blueprint / cluster template
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/85af32f2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/85af32f2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/85af32f2
Branch: refs/heads/branch-2.5
Commit: 85af32f2df56516889cd00267877cc2489e2442f
Parents: f5eaa81
Author: lpuskas <lp...@apache.org>
Authored: Mon Mar 27 20:31:31 2017 +0200
Committer: lpuskas <lp...@apache.org>
Committed: Thu Mar 30 17:02:53 2017 +0200
----------------------------------------------------------------------
.../internal/ProvisionClusterRequest.java | 4 +-
.../validators/HiveServiceValidator.java | 70 ++++++++
.../internal/ProvisionClusterRequestTest.java | 10 +-
.../validators/HiveServiceValidatorTest.java | 158 +++++++++++++++++++
4 files changed, 236 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/85af32f2/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
index a63013a..1a14b01 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java
@@ -38,6 +38,7 @@ import org.apache.ambari.server.topology.NoSuchBlueprintException;
import org.apache.ambari.server.topology.SecurityConfiguration;
import org.apache.ambari.server.topology.TopologyValidator;
import org.apache.ambari.server.topology.validators.ClusterConfigTypeValidator;
+import org.apache.ambari.server.topology.validators.HiveServiceValidator;
import org.apache.ambari.server.topology.validators.RequiredPasswordValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -197,7 +198,8 @@ public class ProvisionClusterRequest extends BaseClusterRequest {
throw new InvalidTopologyTemplateException("Invalid quick links profile", ex);
}
- topologyValidators = ImmutableList.of(new RequiredPasswordValidator(defaultPassword), new ClusterConfigTypeValidator());
+ topologyValidators = ImmutableList.of(new RequiredPasswordValidator(defaultPassword),
+ new ClusterConfigTypeValidator(), new HiveServiceValidator());
}
private String processQuickLinksProfile(Map<String, Object> properties) throws QuickLinksProfileEvaluationException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/85af32f2/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java
new file mode 100644
index 0000000..1351739
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology.validators;
+
+import org.apache.ambari.server.topology.ClusterTopology;
+import org.apache.ambari.server.topology.Configuration;
+import org.apache.ambari.server.topology.InvalidTopologyException;
+import org.apache.ambari.server.topology.TopologyValidator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Validates hive service related setup before provisioning the cluster.
+ */
+public class HiveServiceValidator implements TopologyValidator {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(HiveServiceValidator.class);
+ private static final String HIVE_ENV = "hive-env";
+ private static final String HIVE_DB_DEFAULT = "New MySQL Database";
+ private static final String HIVE_DB_PROPERTY = "hive_database";
+ private static final String MYSQL_SERVER_COMPONENT = "MYSQL_SERVER";
+ public static final String HIVE_SERVICE = "HIVE";
+
+
+ @Override
+ public void validate(ClusterTopology topology) throws InvalidTopologyException {
+
+ // there is no hive configured in the blueprint, nothing to do (does the validator apply?)
+ if (!topology.getBlueprint().getServices().contains(HIVE_SERVICE)) {
+ LOGGER.info(" [{}] service is not listed in the blueprint, skipping hive service validation.", HIVE_SERVICE);
+ return;
+ }
+
+ Configuration clusterConfiguration = topology.getConfiguration();
+
+ // hive database settings are missing (this should never be the case, defaults come from the stack def.)
+ if (!clusterConfiguration.getAllConfigTypes().contains(HIVE_ENV)) {
+ String errorMessage = String.format(" [ %s ] config type is missing from the service [ %s ]. HIVE service validation failed.", HIVE_ENV, HIVE_SERVICE);
+ LOGGER.error(errorMessage);
+ throw new InvalidTopologyException(errorMessage);
+ }
+
+ // hive database has custom configuration, skipping validation
+ if (!HIVE_DB_DEFAULT.equals(clusterConfiguration.getPropertyValue(HIVE_ENV, HIVE_DB_PROPERTY))) {
+ LOGGER.info("Custom hive database settings detected. HIVE service validation succeeded.");
+ return;
+ }
+
+ // hive database settings need the mysql-server component in the blueprint
+ if (!topology.getBlueprint().getServices().contains(MYSQL_SERVER_COMPONENT)) {
+ String errorMessage = String.format("Component [%s] must explicitly be set in the blueprint when hive database " +
+ "is configured with the current settings. HIVE service validation failed.", MYSQL_SERVER_COMPONENT);
+ LOGGER.error(errorMessage);
+ throw new InvalidTopologyException(errorMessage);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/85af32f2/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequestTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequestTest.java
index c78910d..85826f2 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequestTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequestTest.java
@@ -112,7 +112,7 @@ public class ProvisionClusterRequestTest {
assertSame(blueprint, provisionClusterRequest.getBlueprint());
Map<String, HostGroupInfo> hostGroupInfo = provisionClusterRequest.getHostGroupInfo();
assertEquals(1, hostGroupInfo.size());
- assertEquals(2, provisionClusterRequest.getTopologyValidators().size());
+ assertEquals(3, provisionClusterRequest.getTopologyValidators().size());
// group1
// host info
@@ -164,7 +164,7 @@ public class ProvisionClusterRequestTest {
assertSame(blueprint, provisionClusterRequest.getBlueprint());
Map<String, HostGroupInfo> hostGroupInfo = provisionClusterRequest.getHostGroupInfo();
assertEquals(1, hostGroupInfo.size());
- assertEquals(2, provisionClusterRequest.getTopologyValidators().size());
+ assertEquals(3, provisionClusterRequest.getTopologyValidators().size());
// group2
HostGroupInfo group2Info = hostGroupInfo.get("group2");
@@ -216,7 +216,7 @@ public class ProvisionClusterRequestTest {
assertSame(blueprint, provisionClusterRequest.getBlueprint());
Map<String, HostGroupInfo> hostGroupInfo = provisionClusterRequest.getHostGroupInfo();
assertEquals(2, hostGroupInfo.size());
- assertEquals(2, provisionClusterRequest.getTopologyValidators().size());
+ assertEquals(3, provisionClusterRequest.getTopologyValidators().size());
// group1
// host info
@@ -374,7 +374,7 @@ public class ProvisionClusterRequestTest {
TopologyRequest request = new ProvisionClusterRequest(properties, null);
List<TopologyValidator> validators = request.getTopologyValidators();
- assertEquals(2, validators.size());
+ assertEquals(3, validators.size());
TopologyValidator pwdValidator = validators.get(0);
TopologyValidator noDefaultPwdValidator = new RequiredPasswordValidator(null);
@@ -388,7 +388,7 @@ public class ProvisionClusterRequestTest {
TopologyRequest request = new ProvisionClusterRequest(properties, null);
List<TopologyValidator> validators = request.getTopologyValidators();
- assertEquals(2, validators.size());
+ assertEquals(3, validators.size());
TopologyValidator pwdValidator = validators.get(0);
TopologyValidator defaultPwdValidator = new RequiredPasswordValidator("pwd");
http://git-wip-us.apache.org/repos/asf/ambari/blob/85af32f2/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java
new file mode 100644
index 0000000..745b01b
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology.validators;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.ambari.server.topology.Blueprint;
+import org.apache.ambari.server.topology.ClusterTopology;
+import org.apache.ambari.server.topology.Configuration;
+import org.apache.ambari.server.topology.InvalidTopologyException;
+import org.easymock.EasyMock;
+import org.easymock.EasyMockRule;
+import org.easymock.EasyMockSupport;
+import org.easymock.Mock;
+import org.easymock.TestSubject;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class HiveServiceValidatorTest extends EasyMockSupport {
+
+ @Rule
+ public EasyMockRule mocks = new EasyMockRule(this);
+
+ @Mock
+ private ClusterTopology clusterTopologyMock;
+
+ @Mock
+ private Blueprint blueprintMock;
+
+ @Mock
+ private Configuration configurationMock;
+
+ @TestSubject
+ private HiveServiceValidator hiveServiceValidator = new HiveServiceValidator();
+
+ @Before
+ public void setUp() throws Exception {
+
+
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ resetAll();
+ }
+
+ @Test
+ public void testShouldValidationPassWhenHiveServiceIsNotInBlueprint() throws Exception {
+
+ // GIVEN
+ EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock);
+ EasyMock.expect(blueprintMock.getServices()).andReturn(Collections.<String>emptySet());
+ replayAll();
+
+ // WHEN
+ hiveServiceValidator.validate(clusterTopologyMock);
+
+ // THEN
+
+ }
+
+ @Test(expected = InvalidTopologyException.class)
+ public void testShouldValidationFailWhenHiveServiceIsMissingConfigType() throws Exception {
+
+ // GIVEN
+ Collection<String> blueprintServices = Arrays.asList("HIVE", "OOZIE");
+ EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock);
+ EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices);
+ EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock);
+ EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(Collections.<String>emptySet());
+
+ replayAll();
+
+ // WHEN
+ hiveServiceValidator.validate(clusterTopologyMock);
+
+ // THEN
+
+ }
+
+ @Test
+ public void testShouldValidationPassWhenCustomHiveDatabaseSettingsProvided() throws Exception {
+
+ // GIVEN
+ Collection<String> blueprintServices = Arrays.asList("HIVE", "OOZIE");
+ Collection<String> configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env");
+ EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock);
+ EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices);
+ EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock);
+ EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes);
+
+ EasyMock.expect(configurationMock.getPropertyValue("hive-env", "hive_database")).andReturn("PSQL");
+ replayAll();
+
+ // WHEN
+ hiveServiceValidator.validate(clusterTopologyMock);
+
+ // THEN
+
+ }
+
+ @Test(expected = InvalidTopologyException.class)
+ public void testShouldValidationFailWhenDefaultsAreUsedAndMysqlComponentIsMissing() throws Exception {
+ // GIVEN
+ Collection<String> blueprintServices = Arrays.asList("HIVE", "HDFS");
+ Collection<String> configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env");
+ EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes();
+ EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices).anyTimes();
+ EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock);
+ EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes);
+
+ EasyMock.expect(configurationMock.getPropertyValue("hive-env", "hive_database")).andReturn("New MySQL Database");
+ replayAll();
+
+
+ // WHEN
+ hiveServiceValidator.validate(clusterTopologyMock);
+
+ // THEN
+
+ }
+
+ @Test
+ public void testShouldValidationPassWhenDefaultsAreUsedAndMsqlComponentIsListed() throws Exception {
+ // GIVEN
+ Collection<String> blueprintServices = Arrays.asList("HIVE", "HDFS", "MYSQL_SERVER");
+ Collection<String> configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env");
+ EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes();
+ EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices).anyTimes();
+ EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock);
+ EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes);
+
+ EasyMock.expect(configurationMock.getPropertyValue("hive-env", "hive_database")).andReturn("New MySQL Database");
+ replayAll();
+
+ // WHEN
+ hiveServiceValidator.validate(clusterTopologyMock);
+
+ // THEN
+
+ }
+}
\ No newline at end of file