You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2016/02/22 15:27:39 UTC
[2/2] ambari git commit: AMBARI-15006. Ambari Server Upgrade adds
unneeded Atlas properties even though Atlas is not needed,
causing forced Hive restart (aonishuk)
AMBARI-15006. Ambari Server Upgrade adds unneeded Atlas properties even though Atlas is not needed, causing forced Hive restart (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/37b86ee7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/37b86ee7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/37b86ee7
Branch: refs/heads/branch-2.2
Commit: 37b86ee7a2c3cfb79c20c5f7d2aa6e3a28b82a19
Parents: 105ae99
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Feb 22 16:27:28 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Feb 22 16:27:28 2016 +0200
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog222.java | 50 +++++++++++++
.../HIVE/0.12.0.2.0/configuration/hive-site.xml | 1 +
.../services/HIVE/configuration/hive-site.xml | 2 +
.../server/upgrade/UpgradeCatalog222Test.java | 79 ++++++++++++++++++++
4 files changed, 132 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/37b86ee7/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
index 0aa1e7a..2d0b556 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
@@ -28,11 +28,15 @@ import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.utils.VersionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.SQLException;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -49,6 +53,16 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
*/
private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog222.class);
private static final String AMS_SITE = "ams-site";
+ private static final String HIVE_SITE_CONFIG = "hive-site";
+ private static final String ATLAS_APPLICATION_PROPERTIES_CONFIG = "application-properties";
+ private static final String ATLAS_HOOK_HIVE_MINTHREADS_PROPERTY = "atlas.hook.hive.minThreads";
+ private static final String ATLAS_HOOK_HIVE_MAXTHREADS_PROPERTY = "atlas.hook.hive.maxThreads";
+ private static final String ATLAS_CLUSTER_NAME_PROPERTY = "atlas.cluster.name";
+ private static final String ATLAS_ENABLETLS_PROPERTY = "atlas.enableTLS";
+ private static final String ATLAS_SERVER_HTTP_PORT_PROPERTY = "atlas.server.http.port";
+ private static final String ATLAS_SERVER_HTTPS_PORT_PROPERTY = "atlas.server.https.port";
+ private static final String ATLAS_REST_ADDRESS_PROPERTY = "atlas.rest.address";
+
private static final String HOST_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER =
"timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier";
private static final String CLUSTER_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER =
@@ -115,6 +129,7 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
updateAlerts();
updateStormConfigs();
updateAMSConfigs();
+ updateHiveConfig();
updateHostRoleCommands();
}
@@ -285,6 +300,41 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
}
}
+ protected void updateHiveConfig() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) {
+ Config hiveSiteConfig = cluster.getDesiredConfigByType(HIVE_SITE_CONFIG);
+ Config atlasConfig = cluster.getDesiredConfigByType(ATLAS_APPLICATION_PROPERTIES_CONFIG);
+
+ StackId stackId = cluster.getCurrentStackVersion();
+ boolean isStackNotLess23 = (stackId != null && stackId.getStackName().equals("HDP") &&
+ VersionUtils.compareVersions(stackId.getStackVersion(), "2.3") >= 0);
+
+ List<ServiceComponentHost> atlasHost = cluster.getServiceComponentHosts("ATLAS", "ATLAS_SERVER");
+ Map<String, String> updates = new HashMap<String, String>();
+
+ if (isStackNotLess23 && atlasHost.size() != 0 && hiveSiteConfig != null) {
+
+ updates.put(ATLAS_HOOK_HIVE_MINTHREADS_PROPERTY, "1");
+ updates.put(ATLAS_HOOK_HIVE_MAXTHREADS_PROPERTY, "1");
+ updates.put(ATLAS_CLUSTER_NAME_PROPERTY, "primary");
+
+ if (atlasConfig != null && atlasConfig.getProperties().containsKey(ATLAS_ENABLETLS_PROPERTY)) {
+ String atlasEnableTLSProperty = atlasConfig.getProperties().get(ATLAS_ENABLETLS_PROPERTY);
+ String atlasScheme = "http";
+ String atlasServerHttpPortProperty = atlasConfig.getProperties().get(ATLAS_SERVER_HTTP_PORT_PROPERTY);
+ if (atlasEnableTLSProperty.toLowerCase().equals("true")) {
+ atlasServerHttpPortProperty = atlasConfig.getProperties().get(ATLAS_SERVER_HTTPS_PORT_PROPERTY);
+ atlasScheme = "https";
+ }
+ updates.put(ATLAS_REST_ADDRESS_PROPERTY, String.format("%s://%s:%s", atlasScheme, atlasHost.get(0).getHostName(), atlasServerHttpPortProperty));
+ }
+ updateConfigurationPropertiesForCluster(cluster, HIVE_SITE_CONFIG, updates, false, false);
+ }
+ }
+ }
+
+
private String convertToDaysIfInSeconds(String secondsString) {
int seconds = Integer.valueOf(secondsString);
http://git-wip-us.apache.org/repos/asf/ambari/blob/37b86ee7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
index dde726a..078900e 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
@@ -417,6 +417,7 @@ limitations under the License.
<property>
<name>atlas.cluster.name</name>
+ <property-type>DONT_ADD_ON_UPGRADE</property-type>
<value>primary</value>
<depends-on>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37b86ee7/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
index a611386..de2813c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml
@@ -42,6 +42,7 @@ limitations under the License.
<property>
<name>atlas.hook.hive.minThreads</name>
+ <property-type>DONT_ADD_ON_UPGRADE</property-type>
<value>1</value>
<description>
Minimum number of threads maintained by Atlas hook.
@@ -49,6 +50,7 @@ limitations under the License.
</property>
<property>
+ <property-type>DONT_ADD_ON_UPGRADE</property-type>
<name>atlas.hook.hive.maxThreads</name>
<value>1</value>
<description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/37b86ee7/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
index be95149..08b38e3 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
@@ -20,7 +20,9 @@ package org.apache.ambari.server.upgrade;
import java.lang.reflect.Method;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import javax.persistence.EntityManager;
@@ -41,6 +43,8 @@ import org.apache.ambari.server.orm.entities.StackEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.stack.OsFamily;
import org.easymock.Capture;
import org.easymock.EasyMock;
@@ -51,12 +55,14 @@ import org.junit.Test;
import com.google.common.collect.Maps;
import com.google.gson.Gson;
+import com.google.inject.AbstractModule;
import com.google.inject.Binder;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.Provider;
import com.google.inject.persist.PersistService;
+import org.apache.ambari.server.AmbariException;
import static org.easymock.EasyMock.capture;
import static org.easymock.EasyMock.createMockBuilder;
@@ -106,6 +112,7 @@ public class UpgradeCatalog222Test {
Method updateAlerts = UpgradeCatalog222.class.getDeclaredMethod("updateAlerts");
Method updateStormConfigs = UpgradeCatalog222.class.getDeclaredMethod("updateStormConfigs");
Method updateAMSConfigs = UpgradeCatalog222.class.getDeclaredMethod("updateAMSConfigs");
+ Method updateHiveConfigs = UpgradeCatalog222.class.getDeclaredMethod("updateHiveConfig");
Method updateHostRoleCommands = UpgradeCatalog222.class.getDeclaredMethod("updateHostRoleCommands");
@@ -114,6 +121,7 @@ public class UpgradeCatalog222Test {
.addMockedMethod(updateAlerts)
.addMockedMethod(updateStormConfigs)
.addMockedMethod(updateAMSConfigs)
+ .addMockedMethod(updateHiveConfigs)
.addMockedMethod(updateHostRoleCommands)
.createMock();
@@ -127,6 +135,8 @@ public class UpgradeCatalog222Test {
expectLastCall().once();
upgradeCatalog222.updateHostRoleCommands();
expectLastCall().once();
+ upgradeCatalog222.updateHiveConfig();
+ expectLastCall().once();
replay(upgradeCatalog222);
@@ -136,6 +146,75 @@ public class UpgradeCatalog222Test {
}
@Test
+ public void testHiveSiteUpdateConfigs() throws AmbariException {
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+ final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class);
+ final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class);
+ final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class);
+
+ final Config hiveSiteConfigs = easyMockSupport.createNiceMock(Config.class);
+ final Config AtlasSiteConfigs = easyMockSupport.createNiceMock(Config.class);
+
+ final ServiceComponentHost atlasHost = easyMockSupport.createNiceMock(ServiceComponentHost.class);
+ final List<ServiceComponentHost> atlasHosts = new ArrayList<>();
+ atlasHosts.add(atlasHost);
+
+ StackId stackId = new StackId("HDP","2.3");
+
+ final Map<String, String> propertiesAtlasSiteConfigs = new HashMap<String, String>() {{
+ put("atlas.enableTLS", "true");
+ put("atlas.server.https.port", "21443");
+ }};
+
+ final Injector mockInjector = Guice.createInjector(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(AmbariManagementController.class).toInstance(mockAmbariManagementController);
+ bind(Clusters.class).toInstance(mockClusters);
+ bind(EntityManager.class).toInstance(entityManager);
+ bind(ServiceComponentHost.class).toInstance(atlasHost);
+
+ bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
+ bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+ }
+ });
+
+ expect(mockClusterExpected.getCurrentStackVersion()).andReturn(stackId).once();
+ expect(mockClusterExpected.getServiceComponentHosts("ATLAS", "ATLAS_SERVER")).andReturn(atlasHosts).once();
+ expect(atlasHost.getHostName()).andReturn("c6401").once();
+ expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once();
+ expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", mockClusterExpected);
+ }}).atLeastOnce();
+ expect(mockClusterExpected.getDesiredConfigByType("hive-site")).andReturn(hiveSiteConfigs).atLeastOnce();
+ expect(mockClusterExpected.getDesiredConfigByType("application-properties")).andReturn(AtlasSiteConfigs).anyTimes();
+ expect(AtlasSiteConfigs.getProperties()).andReturn(propertiesAtlasSiteConfigs).anyTimes();
+
+ UpgradeCatalog222 upgradeCatalog222 = createMockBuilder(UpgradeCatalog222.class)
+ .withConstructor(Injector.class)
+ .withArgs(mockInjector)
+ .addMockedMethod("updateConfigurationPropertiesForCluster", Cluster.class, String.class,
+ Map.class, boolean.class, boolean.class)
+ .createMock();
+
+ Map<String, String> expectedUpdates = new HashMap<>();
+ expectedUpdates.put("atlas.hook.hive.minThreads", "1");
+ expectedUpdates.put("atlas.hook.hive.maxThreads", "1");
+ expectedUpdates.put("atlas.cluster.name", "primary");
+ expectedUpdates.put("atlas.rest.address", "https://c6401:21443");
+
+ upgradeCatalog222.updateConfigurationPropertiesForCluster(mockClusterExpected, "hive-site", expectedUpdates,
+ false, false);
+ expectLastCall().once();
+
+ easyMockSupport.replayAll();
+ replay(upgradeCatalog222);
+ upgradeCatalog222.updateHiveConfig();
+ easyMockSupport.verifyAll();
+ }
+
+
+ @Test
public void testAmsSiteUpdateConfigs() throws Exception{
Map<String, String> oldPropertiesAmsSite = new HashMap<String, String>() {