You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vb...@apache.org on 2017/07/18 13:57:02 UTC
ambari git commit: AMBARI-21504. Restart of MR2 History Server failed
due to null in immutable_paths.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 d8a5bad1b -> 74cab5067
AMBARI-21504. Restart of MR2 History Server failed due to null in immutable_paths.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/74cab506
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/74cab506
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/74cab506
Branch: refs/heads/branch-2.5
Commit: 74cab506783515f8b6ce62936b990277a6d26a95
Parents: d8a5bad
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Tue Jul 18 16:46:54 2017 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Tue Jul 18 16:46:54 2017 +0300
----------------------------------------------------------------------
.../AmbariCustomCommandExecutionHelper.java | 7 +++++--
.../AmbariManagementControllerImpl.java | 3 ++-
.../internal/ClientConfigResourceProvider.java | 3 ++-
.../ambari/server/state/ConfigHelper.java | 17 +++++++++++++++
.../HDFS/2.1.0.2.0/package/scripts/namenode.py | 4 ++++
.../YARN/2.1.0.2.0/package/scripts/service.py | 4 ++++
.../AmbariManagementControllerImplTest.java | 12 ++++++++++-
.../ambari/server/state/ConfigHelperTest.java | 22 ++++++++++++++++++++
8 files changed, 67 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index e321559..2eca76c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -90,6 +90,7 @@ import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.HostComponentAdminState;
import org.apache.ambari.server.state.HostState;
import org.apache.ambari.server.state.MaintenanceState;
+import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.PropertyInfo.PropertyType;
import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.RepositoryVersionState;
@@ -421,7 +422,8 @@ public class AmbariCustomCommandExecutionHelper {
String groupList = gson.toJson(groupSet);
hostLevelParams.put(GROUP_LIST, groupList);
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+ Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+ Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
@@ -1342,7 +1344,8 @@ public class AmbariCustomCommandExecutionHelper {
hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount());
Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs();
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+ Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+ Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index a34422d..b80ee98 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2541,7 +2541,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
String groupList = gson.toJson(groupSet);
hostParams.put(GROUP_LIST, groupList);
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.NOT_MANAGED_HDFS_PATH, cluster, clusterDesiredConfigs, servicesMap, stackProperties);
+ Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(PropertyType.NOT_MANAGED_HDFS_PATH, cluster, clusterDesiredConfigs, servicesMap, stackProperties);
+ Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index daca2d0..fb90e15 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -409,7 +409,8 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
String groupList = gson.toJson(groupSet);
hostLevelParams.put(GROUP_LIST, groupList);
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs);
+ Map<org.apache.ambari.server.state.PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs);
+ Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index ab8026c..a4c5511 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -220,6 +220,23 @@ public class ConfigHelper {
return resolved;
}
+
+ public Set<String> filterInvalidPropertyValues(Map<PropertyInfo, String> properties, String filteredListName) {
+ Set<String> resultSet = new HashSet<>();
+ for (Iterator<Entry<PropertyInfo, String>> iterator = properties.entrySet().iterator(); iterator.hasNext();) {
+ Entry<PropertyInfo, String> property = iterator.next();
+ PropertyInfo propertyInfo = property.getKey();
+ String propertyValue = property.getValue();
+ if (property == null || propertyValue == null || propertyValue.toLowerCase().equals("null") || propertyValue.isEmpty()) {
+ LOG.error(String.format("Excluding property %s from %s, because of invalid or empty value!", propertyInfo.getName(), filteredListName));
+ iterator.remove();
+ } else {
+ resultSet.add(propertyValue);
+ }
+ }
+ return resultSet;
+ }
+
/**
* Get all config properties for a cluster given a set of configType to
* versionTags map. This helper method merges all the override tags with a
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
index 9a9f9ca..7f7e30c 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
@@ -96,6 +96,10 @@ class NameNode(Script):
env.set_params(params)
self.configure(env)
hdfs_binary = self.get_hdfs_binary()
+
+ if not params.hdfs_tmp_dir or params.hdfs_tmp_dir == None or params.hdfs_tmp_dir.lower() == 'null':
+ Logger.error("WARNING: HDFS tmp dir property (hdfs_tmp_dir) is empty or invalid. Ambari will change permissions for the folder on regular basis.")
+
namenode(action="start", hdfs_binary=hdfs_binary, upgrade_type=upgrade_type,
upgrade_suspended=params.upgrade_suspended, env=env)
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py
index 6495209..deb9e64 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py
@@ -25,6 +25,7 @@ from ambari_commons import OSConst
from resource_management.core.shell import as_user, as_sudo
from resource_management.libraries.functions.show_logs import show_logs
from resource_management.core.signal_utils import TerminateStrategy
+from resource_management.core.logger import Logger
@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
def service(componentName, action='start', serviceName='yarn'):
@@ -42,6 +43,9 @@ def service(componentName, action='start', serviceName='yarn'):
import params
if serviceName == 'mapreduce' and componentName == 'historyserver':
+ if not params.hdfs_tmp_dir or params.hdfs_tmp_dir == None or params.hdfs_tmp_dir.lower() == 'null':
+ Logger.error("WARNING: HDFS tmp dir property (hdfs_tmp_dir) is empty or invalid. Ambari will change permissions for the folder on regular basis.")
+
delete_pid_file = True
daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh")
pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid")
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
index dbfee41..085b746 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
@@ -2055,6 +2055,14 @@ public class AmbariManagementControllerImplTest {
String JCE_NAME = "jceName";
String OJDBC_JAR_NAME = "OjdbcJarName";
String SERVER_DB_NAME = "ServerDBName";
+ Map<PropertyInfo, String> notManagedHdfsPathMap = new HashMap<>();
+ PropertyInfo propertyInfo1 = new PropertyInfo();
+ propertyInfo1.setName("1");
+ PropertyInfo propertyInfo2 = new PropertyInfo();
+ propertyInfo2.setName("2");
+ notManagedHdfsPathMap.put(propertyInfo1, "/tmp");
+ notManagedHdfsPathMap.put(propertyInfo2, "/apps/falcon");
+
Set<String> notManagedHdfsPathSet = new HashSet<>(Arrays.asList("/tmp", "/apps/falcon"));
Gson gson = new Gson();
@@ -2088,8 +2096,10 @@ public class AmbariManagementControllerImplTest {
expect(clusterVersionDAO.findByClusterAndStateCurrent(clusterName)).andReturn(clusterVersionEntity).anyTimes();
expect(clusterVersionEntity.getRepositoryVersion()).andReturn(repositoryVersionEntity).anyTimes();
expect(repositoryVersionEntity.getVersion()).andReturn("1234").anyTimes();
- expect(configHelper.getPropertyValuesWithPropertyType(stackId,
+ expect(configHelper.getPropertiesWithPropertyType(stackId,
PropertyInfo.PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs)).andReturn(
+ notManagedHdfsPathMap);
+ expect(configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST)).andReturn(
notManagedHdfsPathSet);
replay(manager, clusters, cluster, injector, stackId, configuration, clusterVersionDAO, clusterVersionEntity,
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
index b1c10f5..435b246 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
@@ -753,6 +753,28 @@ public class ConfigHelperTest {
}
@Test
+ public void testFilterInvalidPropertyValues() {
+ Map<PropertyInfo, String> properties = new HashMap<>();
+ PropertyInfo prop1 = new PropertyInfo();
+ prop1.setName("1");
+ PropertyInfo prop2 = new PropertyInfo();
+ prop1.setName("2");
+ PropertyInfo prop3 = new PropertyInfo();
+ prop1.setName("3");
+ PropertyInfo prop4 = new PropertyInfo();
+ prop1.setName("4");
+
+ properties.put(prop1, "/tmp");
+ properties.put(prop2, "null");
+ properties.put(prop3, "");
+ properties.put(prop4, null);
+
+ Set<String> resultSet = configHelper.filterInvalidPropertyValues(properties, "testlist");
+ Assert.assertEquals(1, resultSet.size());
+ Assert.assertEquals(resultSet.iterator().next(), "/tmp");
+ }
+
+ @Test
public void testMergeAttributesWithNullProperties() throws Exception {
Map<String, Map<String, String>> persistedAttributes = new HashMap<>();
Map<String, String> persistedFinalAttrs = new HashMap<>();