You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2016/04/09 00:40:35 UTC
ambari git commit: AMBARI-15751. RU/EU PreCheck to ensure
dfs.client.retry.policy.enable is set to false (alejandro)
Repository: ambari
Updated Branches:
refs/heads/trunk 666b3c690 -> faae7ad8f
AMBARI-15751. RU/EU PreCheck to ensure dfs.client.retry.policy.enable is set to false (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/faae7ad8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/faae7ad8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/faae7ad8
Branch: refs/heads/trunk
Commit: faae7ad8f01fe2a8432945d446e4ea1a79b25dcf
Parents: 666b3c6
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Wed Apr 6 18:38:32 2016 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Fri Apr 8 15:40:25 2016 -0700
----------------------------------------------------------------------
.../ambari/server/checks/CheckDescription.java | 2 ++
.../server/checks/ClientRetryPropertyCheck.java | 20 +++++++-----
.../server/upgrade/UpgradeCatalog240.java | 32 ++++++++++++++++++++
.../checks/ClientRetryPropertyCheckTest.java | 30 ++++++++++++------
.../server/upgrade/UpgradeCatalog240Test.java | 3 ++
5 files changed, 69 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/faae7ad8/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java
index 79be901..3e957b1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java
@@ -32,6 +32,8 @@ public enum CheckDescription {
CLIENT_RETRY(PrereqCheckType.SERVICE,
"Client Retry Properties",
new HashMap<String, String>() {{
+ put(ClientRetryPropertyCheck.HDFS_CLIENT_RETRY_DISABLED_KEY,
+ "The hdfs-site.xml property dfs.client.retry.policy.enabled should be set to \"false\" to failover quickly.");
put(ClientRetryPropertyCheck.HIVE_CLIENT_RETRY_MISSING_KEY,
"The hive-site.xml property hive.metastore.failure.retries should be set to a positive value.");
put(ClientRetryPropertyCheck.OOZIE_CLIENT_RETRY_MISSING_KEY,
http://git-wip-us.apache.org/repos/asf/ambari/blob/faae7ad8/ambari-server/src/main/java/org/apache/ambari/server/checks/ClientRetryPropertyCheck.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/ClientRetryPropertyCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/ClientRetryPropertyCheck.java
index 257d575..18fb2ad 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/checks/ClientRetryPropertyCheck.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/ClientRetryPropertyCheck.java
@@ -34,12 +34,13 @@ import com.google.inject.Singleton;
/**
* The {@link ClientRetryPropertyCheck} class is used to check that the
- * client retry properties for HDFS, HIVE, and OOZIE are set.
+ * client retry properties for HIVE and OOZIE are set, but not for HDFS.
*/
@Singleton
-@UpgradeCheck(group = UpgradeCheckGroup.CLIENT_RETRY_PROPERTY, required = false)
+@UpgradeCheck(group = UpgradeCheckGroup.CLIENT_RETRY_PROPERTY, required = true)
public class ClientRetryPropertyCheck extends AbstractCheckDescriptor {
+ static final String HDFS_CLIENT_RETRY_DISABLED_KEY = "hdfs.client.retry.enabled.key";
static final String HIVE_CLIENT_RETRY_MISSING_KEY = "hive.client.retry.missing.key";
static final String OOZIE_CLIENT_RETRY_MISSING_KEY = "oozie.client.retry.missing.key";
@@ -68,15 +69,18 @@ public class ClientRetryPropertyCheck extends AbstractCheckDescriptor {
List<String> errorMessages = new ArrayList<String>();
- // We excluded hdfs-site property dfs.client.retry.policy.enabled because default is false, and should remain
- // that way due to *****.
- // check hdfs client property
+ // HDFS needs to actually prevent client retry since that causes them to try too long and not failover quickly.
+ if (services.containsKey("HDFS")) {
+ String clientRetryPolicyEnabled = getProperty(request, "hdfs-site", "dfs.client.retry.policy.enabled");
+ if (null != clientRetryPolicyEnabled && Boolean.parseBoolean(clientRetryPolicyEnabled)) {
+ errorMessages.add(getFailReason(HDFS_CLIENT_RETRY_DISABLED_KEY, prerequisiteCheck, request));
+ prerequisiteCheck.getFailedOn().add("HDFS");
+ }
+ }
// check hive client properties
if (services.containsKey("HIVE")) {
- String hiveClientRetryCount = getProperty(request, "hive-site",
- "hive.metastore.failure.retries");
-
+ String hiveClientRetryCount = getProperty(request, "hive-site", "hive.metastore.failure.retries");
if (null != hiveClientRetryCount && Integer.parseInt(hiveClientRetryCount) <= 0) {
errorMessages.add(getFailReason(HIVE_CLIENT_RETRY_MISSING_KEY, prerequisiteCheck, request));
prerequisiteCheck.getFailedOn().add("HIVE");
http://git-wip-us.apache.org/repos/asf/ambari/blob/faae7ad8/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
index 65dad79..39ccbe0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
@@ -181,6 +181,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
setRoleSortOrder();
addSettingPermission();
addManageUserPersistedDataPermission();
+ updateHDFSConfigs();
updateAMSConfigs();
updateClusterEnv();
updateHostRoleCommandTableDML();
@@ -1023,6 +1024,37 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
dbAccessor.setColumnNullable(HOST_ROLE_COMMAND_TABLE, columnName, false);
}
+ /**
+ * In hdfs-site, set dfs.client.retry.policy.enabled=false
+ * This is needed for Rolling/Express upgrade so that clients don't keep retrying, which exhausts the retries and
+ * doesn't allow for a graceful failover, which is expected.
+ * @throws AmbariException
+ */
+ protected void updateHDFSConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (final Cluster cluster : clusterMap.values()) {
+ Set<String> installedServices = cluster.getServices().keySet();
+
+ if (installedServices.contains("HDFS")) {
+ Config hdfsSite = cluster.getDesiredConfigByType("hdfs-site");
+ if (hdfsSite != null) {
+ String clientRetryPolicyEnabled = hdfsSite.getProperties().get("dfs.client.retry.policy.enabled");
+ if (null != clientRetryPolicyEnabled && Boolean.parseBoolean(clientRetryPolicyEnabled)) {
+ updateConfigurationProperties("hdfs-site", Collections.singletonMap("dfs.client.retry.policy.enabled", "false"), true, false);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
protected void updateAMSConfigs() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
Clusters clusters = ambariManagementController.getClusters();
http://git-wip-us.apache.org/repos/asf/ambari/blob/faae7ad8/ambari-server/src/test/java/org/apache/ambari/server/checks/ClientRetryPropertyCheckTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/ClientRetryPropertyCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/ClientRetryPropertyCheckTest.java
index 7b8239c..c535978 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/checks/ClientRetryPropertyCheckTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/ClientRetryPropertyCheckTest.java
@@ -78,7 +78,7 @@ public class ClientRetryPropertyCheckTest {
// nothing installed
Assert.assertFalse(m_check.isApplicable(request));
- // YARN installed
+ // HDFS installed
services.put("HDFS", Mockito.mock(Service.class));
Assert.assertTrue(m_check.isApplicable(request));
@@ -97,8 +97,6 @@ public class ClientRetryPropertyCheckTest {
Map<String, Service> services = new HashMap<String, Service>();
Mockito.when(cluster.getServices()).thenReturn(services);
- services.put("HDFS", Mockito.mock(Service.class));
-
final DesiredConfig desiredConfig = Mockito.mock(DesiredConfig.class);
Mockito.when(desiredConfig.getTag()).thenReturn("tag");
Map<String, DesiredConfig> configMap = new HashMap<String, DesiredConfig>();
@@ -112,40 +110,52 @@ public class ClientRetryPropertyCheckTest {
final Map<String, String> properties = new HashMap<String, String>();
Mockito.when(config.getProperties()).thenReturn(properties);
+ // Add HDFS
+ services.put("HDFS", Mockito.mock(Service.class));
+
+ // Add property that will fail
+ properties.put("dfs.client.retry.policy.enabled", "true");
+
PrerequisiteCheck check = new PrerequisiteCheck(null, null);
m_check.perform(check, new PrereqCheckRequest("cluster"));
+ Assert.assertEquals(PrereqCheckStatus.FAIL, check.getStatus());
+
+ // Change property to pass
+ properties.put("dfs.client.retry.policy.enabled", "false");
+ check = new PrerequisiteCheck(null, null);
+ m_check.perform(check, new PrereqCheckRequest("cluster"));
Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus());
- // add hive
+ // Add HIVE
services.put("HIVE", Mockito.mock(Service.class));
- // fail with bad property
+ // Fail with bad property
properties.put("hive.metastore.failure.retries", "0");
check = new PrerequisiteCheck(null, null);
m_check.perform(check, new PrereqCheckRequest("cluster"));
Assert.assertEquals(PrereqCheckStatus.FAIL, check.getStatus());
- // add hive retry
+ // Add hive retry in order to pass
properties.put("hive.metastore.failure.retries", "5");
check = new PrerequisiteCheck(null, null);
m_check.perform(check, new PrereqCheckRequest("cluster"));
Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus());
- // add oozie
+ // Add OOZIE
services.put("OOZIE", Mockito.mock(Service.class));
- // fail without property
+ // Fail without property
check = new PrerequisiteCheck(null, null);
m_check.perform(check, new PrereqCheckRequest("cluster"));
Assert.assertEquals(PrereqCheckStatus.FAIL, check.getStatus());
- // fail without right property
+ // Fail without right property
properties.put("template", "foo bar baz");
check = new PrerequisiteCheck(null, null);
m_check.perform(check, new PrereqCheckRequest("cluster"));
Assert.assertEquals(PrereqCheckStatus.FAIL, check.getStatus());
- // pass with right property
+ // Pass with right property
properties.put("content", "foo bar baz -Doozie.connection.retry.count=5 foobarbaz");
check = new PrerequisiteCheck(null, null);
m_check.perform(check, new PrereqCheckRequest("cluster"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/faae7ad8/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
index a583d39..e03f353 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
@@ -353,6 +353,7 @@ public class UpgradeCatalog240Test {
Method updateAlerts = UpgradeCatalog240.class.getDeclaredMethod("updateAlerts");
Method addManageUserPersistedDataPermission = UpgradeCatalog240.class.getDeclaredMethod("addManageUserPersistedDataPermission");
Method addSettingPermission = UpgradeCatalog240.class.getDeclaredMethod("addSettingPermission");
+ Method updateHDFSConfigs = UpgradeCatalog240.class.getDeclaredMethod("updateHDFSConfigs");
Method updateAmsConfigs = UpgradeCatalog240.class.getDeclaredMethod("updateAMSConfigs");
Method updateClusterEnv = UpgradeCatalog240.class.getDeclaredMethod("updateClusterEnv");
Method updateHostRoleCommandTableDML = UpgradeCatalog240.class.getDeclaredMethod("updateHostRoleCommandTableDML");
@@ -367,6 +368,7 @@ public class UpgradeCatalog240Test {
.addMockedMethod(updateAlerts)
.addMockedMethod(addSettingPermission)
.addMockedMethod(addManageUserPersistedDataPermission)
+ .addMockedMethod(updateHDFSConfigs)
.addMockedMethod(updateAmsConfigs)
.addMockedMethod(updateClusterEnv)
.addMockedMethod(updateHostRoleCommandTableDML)
@@ -379,6 +381,7 @@ public class UpgradeCatalog240Test {
upgradeCatalog240.updateAlerts();
upgradeCatalog240.addSettingPermission();
upgradeCatalog240.addManageUserPersistedDataPermission();
+ upgradeCatalog240.updateHDFSConfigs();
upgradeCatalog240.updateAMSConfigs();
upgradeCatalog240.updateClusterEnv();
upgradeCatalog240.updateHostRoleCommandTableDML();