You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/07/21 04:38:01 UTC
[01/23] ambari git commit: AMBARI-21504. Restart of MR2 History
Server failed due to null in immutable_paths.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-21450 81035d86c -> 23db89b4a
AMBARI-21504. Restart of MR2 History Server failed due to null in immutable_paths.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/74cab506
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/74cab506
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/74cab506
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 74cab506783515f8b6ce62936b990277a6d26a95
Parents: d8a5bad
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Tue Jul 18 16:46:54 2017 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Tue Jul 18 16:46:54 2017 +0300
----------------------------------------------------------------------
.../AmbariCustomCommandExecutionHelper.java | 7 +++++--
.../AmbariManagementControllerImpl.java | 3 ++-
.../internal/ClientConfigResourceProvider.java | 3 ++-
.../ambari/server/state/ConfigHelper.java | 17 +++++++++++++++
.../HDFS/2.1.0.2.0/package/scripts/namenode.py | 4 ++++
.../YARN/2.1.0.2.0/package/scripts/service.py | 4 ++++
.../AmbariManagementControllerImplTest.java | 12 ++++++++++-
.../ambari/server/state/ConfigHelperTest.java | 22 ++++++++++++++++++++
8 files changed, 67 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index e321559..2eca76c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -90,6 +90,7 @@ import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.HostComponentAdminState;
import org.apache.ambari.server.state.HostState;
import org.apache.ambari.server.state.MaintenanceState;
+import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.PropertyInfo.PropertyType;
import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.RepositoryVersionState;
@@ -421,7 +422,8 @@ public class AmbariCustomCommandExecutionHelper {
String groupList = gson.toJson(groupSet);
hostLevelParams.put(GROUP_LIST, groupList);
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+ Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+ Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
@@ -1342,7 +1344,8 @@ public class AmbariCustomCommandExecutionHelper {
hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount());
Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs();
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+ Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+ Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index a34422d..b80ee98 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2541,7 +2541,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
String groupList = gson.toJson(groupSet);
hostParams.put(GROUP_LIST, groupList);
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.NOT_MANAGED_HDFS_PATH, cluster, clusterDesiredConfigs, servicesMap, stackProperties);
+ Map<PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(PropertyType.NOT_MANAGED_HDFS_PATH, cluster, clusterDesiredConfigs, servicesMap, stackProperties);
+ Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index daca2d0..fb90e15 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -409,7 +409,8 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
String groupList = gson.toJson(groupSet);
hostLevelParams.put(GROUP_LIST, groupList);
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs);
+ Map<org.apache.ambari.server.state.PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs);
+ Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index ab8026c..a4c5511 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -220,6 +220,23 @@ public class ConfigHelper {
return resolved;
}
+
+ public Set<String> filterInvalidPropertyValues(Map<PropertyInfo, String> properties, String filteredListName) {
+ Set<String> resultSet = new HashSet<>();
+ for (Iterator<Entry<PropertyInfo, String>> iterator = properties.entrySet().iterator(); iterator.hasNext();) {
+ Entry<PropertyInfo, String> property = iterator.next();
+ PropertyInfo propertyInfo = property.getKey();
+ String propertyValue = property.getValue();
+ if (property == null || propertyValue == null || propertyValue.toLowerCase().equals("null") || propertyValue.isEmpty()) {
+ LOG.error(String.format("Excluding property %s from %s, because of invalid or empty value!", propertyInfo.getName(), filteredListName));
+ iterator.remove();
+ } else {
+ resultSet.add(propertyValue);
+ }
+ }
+ return resultSet;
+ }
+
/**
* Get all config properties for a cluster given a set of configType to
* versionTags map. This helper method merges all the override tags with a
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
index 9a9f9ca..7f7e30c 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
@@ -96,6 +96,10 @@ class NameNode(Script):
env.set_params(params)
self.configure(env)
hdfs_binary = self.get_hdfs_binary()
+
+ if not params.hdfs_tmp_dir or params.hdfs_tmp_dir == None or params.hdfs_tmp_dir.lower() == 'null':
+ Logger.error("WARNING: HDFS tmp dir property (hdfs_tmp_dir) is empty or invalid. Ambari will change permissions for the folder on regular basis.")
+
namenode(action="start", hdfs_binary=hdfs_binary, upgrade_type=upgrade_type,
upgrade_suspended=params.upgrade_suspended, env=env)
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py
index 6495209..deb9e64 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service.py
@@ -25,6 +25,7 @@ from ambari_commons import OSConst
from resource_management.core.shell import as_user, as_sudo
from resource_management.libraries.functions.show_logs import show_logs
from resource_management.core.signal_utils import TerminateStrategy
+from resource_management.core.logger import Logger
@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
def service(componentName, action='start', serviceName='yarn'):
@@ -42,6 +43,9 @@ def service(componentName, action='start', serviceName='yarn'):
import params
if serviceName == 'mapreduce' and componentName == 'historyserver':
+ if not params.hdfs_tmp_dir or params.hdfs_tmp_dir == None or params.hdfs_tmp_dir.lower() == 'null':
+ Logger.error("WARNING: HDFS tmp dir property (hdfs_tmp_dir) is empty or invalid. Ambari will change permissions for the folder on regular basis.")
+
delete_pid_file = True
daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh")
pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-{componentName}.pid")
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
index dbfee41..085b746 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
@@ -2055,6 +2055,14 @@ public class AmbariManagementControllerImplTest {
String JCE_NAME = "jceName";
String OJDBC_JAR_NAME = "OjdbcJarName";
String SERVER_DB_NAME = "ServerDBName";
+ Map<PropertyInfo, String> notManagedHdfsPathMap = new HashMap<>();
+ PropertyInfo propertyInfo1 = new PropertyInfo();
+ propertyInfo1.setName("1");
+ PropertyInfo propertyInfo2 = new PropertyInfo();
+ propertyInfo2.setName("2");
+ notManagedHdfsPathMap.put(propertyInfo1, "/tmp");
+ notManagedHdfsPathMap.put(propertyInfo2, "/apps/falcon");
+
Set<String> notManagedHdfsPathSet = new HashSet<>(Arrays.asList("/tmp", "/apps/falcon"));
Gson gson = new Gson();
@@ -2088,8 +2096,10 @@ public class AmbariManagementControllerImplTest {
expect(clusterVersionDAO.findByClusterAndStateCurrent(clusterName)).andReturn(clusterVersionEntity).anyTimes();
expect(clusterVersionEntity.getRepositoryVersion()).andReturn(repositoryVersionEntity).anyTimes();
expect(repositoryVersionEntity.getVersion()).andReturn("1234").anyTimes();
- expect(configHelper.getPropertyValuesWithPropertyType(stackId,
+ expect(configHelper.getPropertiesWithPropertyType(stackId,
PropertyInfo.PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs)).andReturn(
+ notManagedHdfsPathMap);
+ expect(configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST)).andReturn(
notManagedHdfsPathSet);
replay(manager, clusters, cluster, injector, stackId, configuration, clusterVersionDAO, clusterVersionEntity,
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
index b1c10f5..435b246 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
@@ -753,6 +753,28 @@ public class ConfigHelperTest {
}
@Test
+ public void testFilterInvalidPropertyValues() {
+ Map<PropertyInfo, String> properties = new HashMap<>();
+ PropertyInfo prop1 = new PropertyInfo();
+ prop1.setName("1");
+ PropertyInfo prop2 = new PropertyInfo();
+ prop1.setName("2");
+ PropertyInfo prop3 = new PropertyInfo();
+ prop1.setName("3");
+ PropertyInfo prop4 = new PropertyInfo();
+ prop1.setName("4");
+
+ properties.put(prop1, "/tmp");
+ properties.put(prop2, "null");
+ properties.put(prop3, "");
+ properties.put(prop4, null);
+
+ Set<String> resultSet = configHelper.filterInvalidPropertyValues(properties, "testlist");
+ Assert.assertEquals(1, resultSet.size());
+ Assert.assertEquals(resultSet.iterator().next(), "/tmp");
+ }
+
+ @Test
public void testMergeAttributesWithNullProperties() throws Exception {
Map<String, Map<String, String>> persistedAttributes = new HashMap<>();
Map<String, String> persistedFinalAttrs = new HashMap<>();
[06/23] ambari git commit: AMBARI-21512. Stack Advisor reported an
error: KeyError: stack_name while Issued INSTALLED as new state for
NODEMANAGER (smohanty)
Posted by jo...@apache.org.
AMBARI-21512. Stack Advisor reported an error: KeyError: stack_name while Issued INSTALLED as new state for NODEMANAGER (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9ac42d0c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9ac42d0c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9ac42d0c
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 9ac42d0c7e52c75e2cd7c6ac65025ae6ee1a83be
Parents: 6d03069
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Jul 18 22:05:46 2017 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Jul 18 22:05:46 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/ambari/server/agent/hello.rs | 0
ambari-server/src/main/resources/stacks/stack_advisor.py | 7 ++++---
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ac42d0c/ambari-server/src/main/java/org/apache/ambari/server/agent/hello.rs
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/hello.rs b/ambari-server/src/main/java/org/apache/ambari/server/agent/hello.rs
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ac42d0c/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 1e0d83a..ccbb676 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -1156,9 +1156,10 @@ class DefaultStackAdvisor(StackAdvisor):
if cluster_env and "stack_root" in cluster_env:
stack_root_as_str = cluster_env["stack_root"]
stack_roots = json.loads(stack_root_as_str)
- stack_name = cluster_env["stack_name"]
- if stack_name in stack_roots:
- stack_root = stack_roots[stack_name]
+ if "stack_name" in cluster_env:
+ stack_name = cluster_env["stack_name"]
+ if stack_name in stack_roots:
+ stack_root = stack_roots[stack_name]
return stack_root
[05/23] ambari git commit: AMBARI-21510. Convert calculated value for
'hive.server2.tez.sessions.per.default.queue' to long before setting it.
Posted by jo...@apache.org.
AMBARI-21510. Convert calculated value for 'hive.server2.tez.sessions.per.default.queue' to long before setting it.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6d030690
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6d030690
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6d030690
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 6d030690bb8056339bc4ac412ca01e2211aaae64
Parents: 512caee
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Tue Jul 18 12:48:53 2017 -0700
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Tue Jul 18 15:31:21 2017 -0700
----------------------------------------------------------------------
.../main/resources/stacks/HDP/2.5/services/stack_advisor.py | 7 +++----
.../src/test/python/stacks/2.5/common/test_stack_advisor.py | 4 ++--
2 files changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d030690/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 9971bfa..855d33ad 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -1254,11 +1254,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if not llap_concurrency_in_changed_configs:
min_llap_concurrency = 1
- putHiveInteractiveSiteProperty('hive.server2.tez.sessions.per.default.queue', llap_concurrency)
- putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "minimum",
- min_llap_concurrency)
+ putHiveInteractiveSiteProperty('hive.server2.tez.sessions.per.default.queue', long(llap_concurrency))
+ putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "minimum", min_llap_concurrency)
- putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "maximum", max_llap_concurreny)
+ putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "maximum", long(max_llap_concurreny))
num_llap_nodes = long(num_llap_nodes)
http://git-wip-us.apache.org/repos/asf/ambari/blob/6d030690/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 222f100..9f75f9d 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -1561,7 +1561,7 @@ class TestHDP25StackAdvisor(TestCase):
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts)
- self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '3.0'})
+ self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '3'})
self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
@@ -2773,7 +2773,7 @@ class TestHDP25StackAdvisor(TestCase):
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts)
self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=2.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=2.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.ordering-policy=priority-utilization\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.roo
t.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.priority=10\nyarn.scheduler.capacity.root.llap.minimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=98.0\nyarn.scheduler.capacity.root.llap.capacity=98.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'})
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1.0')
+ self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1')
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'})
self.assertTrue('num_llap_nodes_for_llap_daemons' not in configurations['hive-interactive-env']['properties'])
[02/23] ambari git commit: AMBARI-21509 - BigInsights/4.2 upgrade to
HDP uses function take_snapshot but that doesn't exist in 4.2's
hbase_upgrade.py
Posted by jo...@apache.org.
AMBARI-21509 - BigInsights/4.2 upgrade to HDP uses function take_snapshot but that doesn't exist in 4.2's hbase_upgrade.py
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cf4888d5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cf4888d5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cf4888d5
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: cf4888d561a09908524d30efde99290a4bc03708
Parents: 74cab50
Author: Tim Thorpe <tt...@apache.org>
Authored: Tue Jul 18 10:50:53 2017 -0700
Committer: Tim Thorpe <tt...@apache.org>
Committed: Tue Jul 18 10:50:53 2017 -0700
----------------------------------------------------------------------
.../4.2/services/HBASE/package/scripts/hbase_upgrade.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cf4888d5/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HBASE/package/scripts/hbase_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HBASE/package/scripts/hbase_upgrade.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HBASE/package/scripts/hbase_upgrade.py
index 610f527..8b1e661 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HBASE/package/scripts/hbase_upgrade.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HBASE/package/scripts/hbase_upgrade.py
@@ -24,7 +24,7 @@ from resource_management.core.resources.system import Execute
class HbaseMasterUpgrade(Script):
- def snapshot(self, env):
+ def take_snapshot(self, env):
import params
snap_cmd = "echo 'snapshot_all' | {0} shell".format(params.hbase_cmd)
[04/23] ambari git commit: AMBARI-21487. Add SOLR to BigInsights
4.2.5 stack (oleewere)
Posted by jo...@apache.org.
AMBARI-21487. Add SOLR to BigInsights 4.2.5 stack (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/512caee1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/512caee1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/512caee1
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 512caee1314ef49e7caa3169b78c125448f310bb
Parents: cf4888d
Author: oleewere <ol...@gmail.com>
Authored: Mon Jul 17 13:58:01 2017 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Tue Jul 18 20:06:34 2017 +0200
----------------------------------------------------------------------
.../BigInsights/4.2.5/services/SOLR/alerts.json | 37 +++
.../SOLR/configuration/ranger-solr-audit.xml | 127 +++++++
.../ranger-solr-plugin-properties.xml | 88 +++++
.../configuration/ranger-solr-policymgr-ssl.xml | 66 ++++
.../SOLR/configuration/ranger-solr-security.xml | 58 ++++
.../SOLR/configuration/solr-client-log4j.xml | 48 +++
.../services/SOLR/configuration/solr-env.xml | 262 +++++++++++++++
.../services/SOLR/configuration/solr-log4j.xml | 36 ++
.../services/SOLR/configuration/solr-xml.xml | 36 ++
.../4.2.5/services/SOLR/kerberos.json | 51 +++
.../4.2.5/services/SOLR/metainfo.xml | 124 +++++++
.../services/SOLR/package/scripts/params.py | 331 +++++++++++++++++++
.../SOLR/package/scripts/service_check.py | 48 +++
.../SOLR/package/scripts/setup_ranger_solr.py | 126 +++++++
.../services/SOLR/package/scripts/setup_solr.py | 129 ++++++++
.../4.2.5/services/SOLR/package/scripts/solr.py | 161 +++++++++
.../SOLR/package/scripts/solr_client.py | 51 +++
.../SOLR/package/scripts/status_params.py | 34 ++
.../SOLR/package/templates/solr_jaas.conf.j2 | 26 ++
.../properties/solr-client-log4j.properties.j2 | 38 +++
.../services/SOLR/properties/solr-env.sh.j2 | 126 +++++++
.../SOLR/properties/solr-log4j.properties.j2 | 40 +++
.../4.2.5/services/SOLR/properties/solr.xml.j2 | 26 ++
.../services/SOLR/quicklinks/quicklinks.json | 37 +++
.../4.2.5/services/SOLR/themes/theme.json | 107 ++++++
25 files changed, 2213 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/alerts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/alerts.json b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/alerts.json
new file mode 100644
index 0000000..30a59fb
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/alerts.json
@@ -0,0 +1,37 @@
+{
+ "SOLR": {
+ "SOLR": [
+ {
+ "name": "solr",
+ "label": "Solr Web UI",
+ "description": "This host-level alert is triggered if the Solr Cloud Instance is unreachable.",
+ "interval": 1,
+ "scope": "ANY",
+ "source": {
+ "type": "WEB",
+ "uri": {
+ "http": "{{solr-env/solr_port}}",
+ "https": "{{solr-env/solr_port}}",
+ "https_property": "{{solr-env/solr_ssl_enabled}}",
+ "https_property_value": "true",
+ "connection_timeout": 5.0,
+ "kerberos_keytab": "{{solr-env/solr_web_kerberos_keytab}}",
+ "kerberos_principal": "{{solr-env/solr_web_kerberos_principal}}",
+ "default_port": 8983
+ },
+ "reporting": {
+ "ok": {
+ "text": "HTTP {0} response in {2:.3f}s"
+ },
+ "warning": {
+ "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
+ },
+ "critical": {
+ "text": "Connection failed to {1} ({3})"
+ }
+ }
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-audit.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-audit.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-audit.xml
new file mode 100644
index 0000000..9fe7af8
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-audit.xml
@@ -0,0 +1,127 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>xasecure.audit.is.enabled</name>
+ <value>true</value>
+ <description>Is Audit enabled?</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.credential.provider.file</name>
+ <value>jceks://file{{credential_file}}</value>
+ <description>Credential file store</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.hdfs</name>
+ <value>true</value>
+ <display-name>Audit to HDFS</display-name>
+ <description>Is Audit to HDFS enabled?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.hdfs</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.hdfs.dir</name>
+ <value>hdfs://NAMENODE_HOSTNAME:8020/ranger/audit</value>
+ <description>HDFS folder to write audit to, make sure the service user has requried permissions</description>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.hdfs.dir</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.hdfs.batch.filespool.dir</name>
+ <value>/var/log/solr/audit/hdfs/spool</value>
+ <description>/var/log/solr/audit/hdfs/spool</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.solr</name>
+ <value>false</value>
+ <display-name>Audit to SOLR</display-name>
+ <description>Is Solr audit enabled?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>xasecure.audit.destination.solr</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.solr.urls</name>
+ <value/>
+ <description>Solr URL</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>ranger-admin-site</type>
+ <name>ranger.audit.solr.urls</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.solr.zookeepers</name>
+ <value>NONE</value>
+ <description>Solr Zookeeper string</description>
+ <depends-on>
+ <property>
+ <type>ranger-admin-site</type>
+ <name>ranger.audit.solr.zookeepers</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.destination.solr.batch.filespool.dir</name>
+ <value>/var/log/solr/audit/solr/spool</value>
+ <description>/var/log/solr/audit/solr/spool</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.audit.provider.summary.enabled</name>
+ <value>true</value>
+ <display-name>Audit provider summary enabled</display-name>
+ <description>Enable Summary audit?</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-plugin-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-plugin-properties.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-plugin-properties.xml
new file mode 100644
index 0000000..77a3219
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-plugin-properties.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+ <property>
+ <name>policy_user</name>
+ <value>ambari-qa</value>
+ <display-name>Policy user for SOLR</display-name>
+ <description>This user must be system user and also present at Ranger admin portal</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hadoop.rpc.protection</name>
+ <value/>
+ <description>Used for repository creation on ranger admin</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>common.name.for.certificate</name>
+ <value/>
+ <description>Common name for certificate, this value should match what is specified in repo within ranger admin</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr.url</name>
+ <value>localhost:8983</value>
+ <description>Used for repository creation on ranger admin</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger-solr-plugin-enabled</name>
+ <value>No</value>
+ <display-name>Enable Ranger for SOLR</display-name>
+ <description>Enable ranger solr plugin</description>
+ <depends-on>
+ <property>
+ <type>ranger-env</type>
+ <name>ranger-solr-plugin-enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <type>boolean</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>REPOSITORY_CONFIG_USERNAME</name>
+ <value>solr</value>
+ <display-name>Ranger repository config user</display-name>
+ <description>Used for repository creation on ranger admin</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>REPOSITORY_CONFIG_PASSWORD</name>
+ <value>solr</value>
+ <property-type>PASSWORD</property-type>
+ <display-name>Ranger repository config password</display-name>
+ <description>Used for repository creation on ranger admin</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-policymgr-ssl.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-policymgr-ssl.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-policymgr-ssl.xml
new file mode 100644
index 0000000..f656364
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-policymgr-ssl.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore</name>
+ <value>solrdev-clientcert.jks</value>
+ <description>Java Keystore files</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore.password</name>
+ <value>myKeyFilePassword</value>
+ <property-type>PASSWORD</property-type>
+ <description>password for keystore</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore</name>
+ <value>cacerts-xasecure.jks</value>
+ <description>java truststore file</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore.password</name>
+ <value>changeit</value>
+ <property-type>PASSWORD</property-type>
+ <description>java truststore password</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore.credential.file</name>
+ <value>jceks://file/{{credential_file}}</value>
+ <description>java keystore credential file</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore.credential.file</name>
+ <value>jceks://file/{{credential_file}}</value>
+ <description>java truststore credential file</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-security.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-security.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-security.xml
new file mode 100644
index 0000000..e3d4cea
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/ranger-solr-security.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>ranger.plugin.solr.service.name</name>
+ <value>{{repo_name}}</value>
+ <description>Name of the Ranger service containing policies for this Solr instance</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.solr.policy.source.impl</name>
+ <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value>
+ <description>Class to retrieve policies from the source</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.solr.policy.rest.url</name>
+ <value>{{policymgr_mgr_url}}</value>
+ <description>URL to Ranger Admin</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.solr.policy.rest.ssl.config.file</name>
+ <value>/etc/solr/conf/ranger-policymgr-ssl.xml</value>
+ <description>Path to the file containing SSL details to contact Ranger Admin</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.solr.policy.pollIntervalMs</name>
+ <value>30000</value>
+ <description>How often to poll for changes in policies?</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger.plugin.solr.policy.cache.dir</name>
+ <value>/etc/ranger/{{repo_name}}/policycache</value>
+ <description>Directory where Ranger policies are cached after successful retrieval from the source</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-client-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-client-log4j.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-client-log4j.xml
new file mode 100644
index 0000000..1257901
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-client-log4j.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<!-- This is a special config file for properties used to monitor status of the service -->
+<configuration supports_adding_forbidden="true">
+ <!-- log4j.xml -->
+ <property>
+ <name>solr_client_log_dir</name>
+ <value>/var/log/solr-client</value>
+ <description>Directory for Solr client logs</description>
+ <display-name>Solr Client log dir</display-name>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>content</name>
+ <display-name>log4j template</display-name>
+ <description>This is the jinja template for log4j.properties file for solr client</description>
+ <value/>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>solr-client-log4j.properties.j2</property-file-name>
+ <property-file-type>text</property-file-type>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-env.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-env.xml
new file mode 100644
index 0000000..4d47cf1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-env.xml
@@ -0,0 +1,262 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<!-- This is a special config file for properties used to monitor status of the service -->
+<configuration supports_adding_forbidden="true">
+ <property>
+ <name>solr_port</name>
+ <value>8983</value>
+ <description>Solr port</description>
+ <display-name>Solr port</display-name>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_jmx_port</name>
+ <value>18983</value>
+ <description>Solr JMX port</description>
+ <display-name>Solr JMX port</display-name>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_pid_dir</name>
+ <value>/var/run/solr</value>
+ <description>Solr Process ID Directory</description>
+ <display-name>Solr pid dir</display-name>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_log_dir</name>
+ <value>/var/log/solr</value>
+ <description>Directory for Solr logs</description>
+ <display-name>Solr log dir</display-name>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_user</name>
+ <value>solr</value>
+ <property-type>USER</property-type>
+ <description>Solr user</description>
+ <display-name>Solr User</display-name>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_datadir</name>
+ <value>/opt/solr/data</value>
+ <display-name>Solr data dir</display-name>
+ <description>Directory for storing Solr index. Make sure you have enough disk space</description>
+ <value-attributes>
+ <type>directory</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_hdfs_home_dir</name>
+ <value>/apps/solr/data</value>
+ <description>A root location in HDFS for Solr to write collection data to. Rather than specifying an HDFS location for the data directory or update log directory, use this to specify one root location and have everything automatically created within this HDFS</description>
+ </property>
+ <property>
+ <name>solr_ssl_enabled</name>
+ <value>false</value>
+ <display-name>Enable SSL to Solr</display-name>
+ <description>Enable ssl to Solr</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_truststore_location</name>
+ <value>/etc/security/serverKeys/solr.trustStore.jks</value>
+ <display-name>Solr trust store location</display-name>
+ <description>Location of the trust store file. (default value is not generated)</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_truststore_type</name>
+ <value>jks</value>
+ <display-name>Solr trust store type</display-name>
+ <description>Type of the trust store file.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_truststore_password</name>
+ <value>bigdata</value>
+ <property-type>PASSWORD</property-type>
+ <display-name>Solr trust store password</display-name>
+ <description>Password to open the trust store file.</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_keystore_location</name>
+ <value>/etc/security/serverKeys/solr.keyStore.jks</value>
+ <display-name>Solr key store location</display-name>
+ <description>Location of the key store file. (default value is not generated)</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_keystore_type</name>
+ <value>jks</value>
+ <display-name>Solr key store type</display-name>
+ <description>Type of the key store file.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_keystore_password</name>
+ <value>bigdata</value>
+ <display-name>Solr key store password</display-name>
+ <property-type>PASSWORD</property-type>
+ <description>Password to open the key store file.</description>
+ <value-attributes>
+ <type>password</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_znode</name>
+ <value>/solr</value>
+ <description>Zookeeper znode, e.g: /solr</description>
+ <display-name>Solr ZNode</display-name>
+ <value-attributes>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_minmem</name>
+ <value>1024</value>
+ <display-name>Solr Minimum Heap Size</display-name>
+ <description>Solr minimum heap size e.g. 512m</description>
+ <value-attributes>
+ <type>int</type>
+ <minimum>512</minimum>
+ <maximum>32768</maximum>
+ <unit>MB</unit>
+ <increment-step>256</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>solr_maxmem</name>
+ <value>2048</value>
+ <display-name>Solr Maximum Heap Size</display-name>
+ <description>Solr maximum heap size e.g. 512m</description>
+ <value-attributes>
+ <type>int</type>
+ <minimum>512</minimum>
+ <maximum>32768</maximum>
+ <unit>MB</unit>
+ <increment-step>256</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>solr_kerberos_keytab</name>
+ <value>/etc/security/keytabs/solr.service.keytab</value>
+ <display-name>Solr keytab</display-name>
+ <description>The path to the Kerberos Keytab file containing service principal of Solr.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>solr_kerberos_principal</name>
+ <value>solr/_HOST@EXAMPLE.COM</value>
+ <display-name>Solr principal</display-name>
+ <description>The service principal for Solr.</description>
+ <property-type>KERBEROS_PRINCIPAL</property-type>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>solr_web_kerberos_keytab</name>
+ <value>/etc/security/keytabs/spnego.service.keytab</value>
+ <display-name>Solr Http keytab</display-name>
+ <description>The path to the Kerberos Keytab file containing service principal of Solr.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>solr_web_kerberos_principal</name>
+ <value>HTTP/_HOST@EXAMPLE.COM</value>
+ <display-name>Solr Http principal</display-name>
+ <description>The service principal for the Solr.</description>
+ <property-type>KERBEROS_PRINCIPAL</property-type>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>solr_kerberos_name_rules</name>
+ <value>DEFAULT</value>
+ <display-name>Solr Kerberos name rules</display-name>
+ <description>Kerberos name rules for Spnego</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!-- solr-env.sh -->
+ <property>
+ <name>content</name>
+ <display-name>solr-env template</display-name>
+ <description>This is the jinja template for solr-env.sh file</description>
+ <value/>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>solr-env.sh.j2</property-file-name>
+ <property-file-type>text</property-file-type>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>is_supported_solr_ranger</name>
+ <value>true</value>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>authorizer.class.name</name>
+ <description>
+ Solr authorizer class
+ </description>
+ <depends-on>
+ <property>
+ <type>ranger-solr-plugin-properties</type>
+ <name>ranger-solr-plugin-enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-log4j.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-log4j.xml
new file mode 100644
index 0000000..5fe54ff
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-log4j.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <property>
+ <name>content</name>
+ <display-name>solr-log4j template</display-name>
+ <description>This is the jinja template for log4j.properties</description>
+ <value/>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>solr-log4j.properties.j2</property-file-name>
+ <property-file-type>text</property-file-type>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-xml.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-xml.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-xml.xml
new file mode 100644
index 0000000..469c2b5
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-xml.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <property>
+ <name>content</name>
+ <display-name>solr-xml template</display-name>
+ <description>This is the jinja template for Solr's solr.xml file</description>
+ <value/>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>solr.xml.j2</property-file-name>
+ <property-file-type>xml</property-file-type>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/kerberos.json b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/kerberos.json
new file mode 100644
index 0000000..84d7fc3
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/kerberos.json
@@ -0,0 +1,51 @@
+{
+ "services": [
+ {
+ "name": "SOLR",
+ "identities": [
+ {
+ "name": "/smokeuser"
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "solr-env/solr_web_kerberos_principal"
+ },
+ "keytab": {
+ "configuration": "solr-env/solr_web_kerberos_keytab"
+ }
+ }
+ ],
+ "components": [
+ {
+ "name": "SOLR",
+ "identities": [
+ {
+ "name": "solr",
+ "principal": {
+ "value": "${solr-env/solr_user}/_HOST@${realm}",
+ "type": "service",
+ "configuration": "solr-env/solr_kerberos_principal"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/solr.service.keytab",
+ "owner": {
+ "name": "${solr-env/solr_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "solr-env/solr_kerberos_keytab"
+ }
+ }
+ ]
+ },
+ {
+ "name": "SOLR_CLIENT"
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/metainfo.xml
new file mode 100644
index 0000000..9fac8a8
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/metainfo.xml
@@ -0,0 +1,124 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>SOLR</name>
+ <displayName>Solr</displayName>
+ <comment>Solr is the popular, blazing fast open source enterprise search platform from the Apache Lucene project</comment>
+ <version>6.3.0</version>
+ <components>
+ <component>
+ <name>SOLR</name>
+ <timelineAppid>solr</timelineAppid>
+ <displayName>Solr</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <versionAdvertised>false</versionAdvertised>
+ <commandScript>
+ <script>scripts/solr.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <logs>
+ <log>
+ <logId>solr</logId>
+ <primary>true</primary>
+ </log>
+ </logs>
+ <dependencies>
+
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+
+ </dependencies>
+ <configuration-dependencies>
+ <config-type>solr-env</config-type>
+ <config-type>solr-xml</config-type>
+ <config-type>solr-log4j</config-type>
+ <config-type>ranger-solr-plugin-properties</config-type>
+ <config-type>ranger-solr-audit</config-type>
+ <config-type>ranger-solr-policymgr-ssl</config-type>
+ <config-type>ranger-solr-security</config-type>
+ </configuration-dependencies>
+ </component>
+
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>redhat7,redhat6,suse11</osFamily>
+ <packages>
+ <package>
+ <name>solr_4_2_5_*</name>
+ </package>
+ <package>
+ <name>ambari-infra-solr-client-*</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>ubuntu12</osFamily>
+ <packages>
+ <package>
+ <name>solr_4_2_5_*</name>
+ </package>
+ <package>
+ <name>ambari-infra-solr-client-*</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>ZOOKEEPER</service>
+ <service>HDFS</service>
+ </requiredServices>
+
+ <themes>
+ <theme>
+ <fileName>theme.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
+ <quickLinksConfigurations>
+ <quickLinksConfiguration>
+ <fileName>quicklinks.json</fileName>
+ <default>true</default>
+ </quickLinksConfiguration>
+ </quickLinksConfigurations>
+
+ <configuration-dependencies>
+ <config-type>ranger-env</config-type>
+ </configuration-dependencies>
+
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/params.py
new file mode 100644
index 0000000..b708026
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/params.py
@@ -0,0 +1,331 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from ambari_commons.constants import AMBARI_SUDO_BINARY
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import Direction
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.resources import HdfsResource
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.stack_features import get_stack_feature_version
+from resource_management.libraries.functions.get_stack_version import get_stack_version
+
+import os
+import status_params
+
+def get_port_from_url(address):
+ if not is_empty(address):
+ return address.split(':')[-1]
+ else:
+ return address
+
+# config object that holds the configurations declared in the -site.xml file
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+stack_root = Script.get_stack_root()
+stack_name = default("/hostLevelParams/stack_name", None)
+retryAble = default("/commandParams/command_retry_enabled", False)
+version = default("/commandParams/version", None)
+
+
+current_version = default("/hostLevelParams/current_version", None)
+upgrade_direction = default("/commandParams/upgrade_direction", None)
+stack_version = default("/commandParams/version", None)
+sudo = AMBARI_SUDO_BINARY
+security_enabled = status_params.security_enabled
+fs_root = config['configurations']['core-site']['fs.defaultFS']
+solr_conf = "/etc/solr/conf"
+
+solr_port = status_params.solr_port
+solr_piddir = status_params.solr_piddir
+solr_pidfile = status_params.solr_pidfile
+
+user_group = config['configurations']['cluster-env']['user_group']
+fetch_nonlocal_groups = config['configurations']['cluster-env']["fetch_nonlocal_groups"]
+
+# shared configs
+java64_home = config['hostLevelParams']['java_home']
+zookeeper_hosts_list = config['clusterHostInfo']['zookeeper_hosts']
+zookeeper_hosts_list.sort()
+# get comma separated list of zookeeper hosts from clusterHostInfo
+zookeeper_hosts = ",".join(zookeeper_hosts_list)
+
+#####################################
+# Solr configs
+#####################################
+
+# Only supporting SolrCloud mode - so hardcode those options
+solr_cloudmode = 'true'
+solr_dir = '/usr/iop/current/solr-server'
+solr_client_dir = '/usr/iop/current/solr-client'
+solr_bindir = solr_dir + '/bin'
+cloud_scripts = solr_dir + '/server/scripts/cloud-scripts'
+
+if "solr-env" in config['configurations']:
+ solr_hosts = config['clusterHostInfo']['solr_hosts']
+ solr_znode = default('/configurations/solr-env/solr_znode', '/solr')
+ solr_min_mem = default('/configurations/solr-env/solr_minmem', 1024)
+ solr_max_mem = default('/configurations/solr-env/solr_maxmem', 2048)
+ solr_instance_count = len(config['clusterHostInfo']['solr_hosts'])
+ solr_datadir = default('/configurations/solr-env/solr_datadir', '/opt/solr/data')
+ solr_data_resources_dir = os.path.join(solr_datadir, 'resources')
+ solr_jmx_port = default('/configurations/solr-env/solr_jmx_port', 18983)
+ solr_ssl_enabled = default('configurations/solr-env/solr_ssl_enabled', False)
+ solr_keystore_location = config['configurations']['solr-env']['solr_keystore_location']
+ solr_keystore_password = config['configurations']['solr-env']['solr_keystore_password']
+ solr_keystore_type = config['configurations']['solr-env']['solr_keystore_type']
+ solr_truststore_location = config['configurations']['solr-env']['solr_truststore_location']
+ solr_truststore_password = config['configurations']['solr-env']['solr_truststore_password']
+ solr_truststore_type = config['configurations']['solr-env']['solr_truststore_type']
+ solr_user = config['configurations']['solr-env']['solr_user']
+ solr_log_dir = config['configurations']['solr-env']['solr_log_dir']
+ solr_log = format("{solr_log_dir}/solr-install.log")
+ solr_env_content = config['configurations']['solr-env']['content']
+ solr_hdfs_home_dir = config['configurations']['solr-env']['solr_hdfs_home_dir']
+
+if upgrade_direction is not None and upgrade_direction == Direction.UPGRADE:
+ old_lib_dir=default("/configurations/solr-env/solr_lib_dir", None)
+
+zookeeper_port = default('/configurations/zoo.cfg/clientPort', None)
+# get comma separated list of zookeeper hosts from clusterHostInfo
+index = 0
+zookeeper_quorum = ""
+for host in config['clusterHostInfo']['zookeeper_hosts']:
+ zookeeper_quorum += host + ":" + str(zookeeper_port)
+ index += 1
+ if index < len(config['clusterHostInfo']['zookeeper_hosts']):
+ zookeeper_quorum += ","
+
+solr_jaas_file = None
+
+if security_enabled:
+ _hostname_lowercase = config['hostname'].lower()
+ solr_jaas_file = solr_conf + '/solr_jaas.conf'
+ solr_kerberos_keytab = default('/configurations/solr-env/solr_kerberos_keytab', None)
+ if not solr_kerberos_keytab: #Maybe against older configurations during a downgrade operation. Look for the old property
+ solr_keytab=config['configurations']['solr-site']['solr.hdfs.security.kerberos.keytabfile']
+ solr_kerberos_keytab = solr_keytab
+
+ solr_kerberos_principal = default('/configurations/solr-env/solr_kerberos_principal', None)
+ if solr_kerberos_principal:
+ solr_kerberos_principal = solr_kerberos_principal.replace('_HOST',_hostname_lowercase)
+ else: #Maybe against older configurations during a downgrade operation. Look for the old property
+ solr_site = dict(config['configurations']['solr-site'])
+ solr_principal = solr_site['solr.hdfs.security.kerberos.principal']
+ solr_principal = solr_principal.replace('_HOST', _hostname_lowercase)
+ solr_site['solr.hdfs.security.kerberos.principal']=solr_principal
+ solr_kerberos_principal = solr_principal
+
+ solr_web_kerberos_keytab = config['configurations']['solr-env']['solr_web_kerberos_keytab']
+ solr_web_kerberos_principal = default('/configurations/solr-env/solr_web_kerberos_principal', None)
+ if solr_web_kerberos_principal:
+ solr_web_kerberos_principal = solr_web_kerberos_principal.replace('_HOST',_hostname_lowercase)
+ solr_kerberos_name_rules = config['configurations']['solr-env']['solr_kerberos_name_rules']
+
+solr_xml_content = default('configurations/solr-xml/content', None)
+solr_log4j_content = default('configurations/solr-log4j/content', None)
+
+solr_client_custom_log4j = "solr-client-log4j" in config['configurations']
+
+restart_during_downgrade = False
+upgrade_direction = default("/commandParams/upgrade_direction", None)
+restart_during_downgrade = (upgrade_direction == Direction.DOWNGRADE)
+
+# *********************** RANGER PLUGIN CHANGES ***********************
+# ranger host
+# **********************************************************************
+
+# get the correct version to use for checking stack features
+version_for_stack_feature_checks = get_stack_feature_version(config)
+
+stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
+stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
+
+ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+has_ranger_admin = not len(ranger_admin_hosts) == 0
+xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+ranger_admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir","/var/log/ranger/admin")
+
+#need to set the defaut to false to satisfy downgrade from 4.2,5 to 4.2 or 4.1
+is_supported_solr_ranger = default('/configurations/solr-env/is_supported_solr_ranger', False)
+
+#ranger solr properties
+if has_ranger_admin and is_supported_solr_ranger:
+
+ enable_ranger_solr = config['configurations']['ranger-solr-plugin-properties']['ranger-solr-plugin-enabled']
+ enable_ranger_solr = not is_empty(enable_ranger_solr) and enable_ranger_solr.lower() == 'yes'
+ policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
+ if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
+ policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
+ xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR']
+ xa_audit_db_flavor = xa_audit_db_flavor.lower() if xa_audit_db_flavor else None
+ xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits')
+ xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger')
+ xa_audit_db_password = ''
+ if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db:
+ xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
+ xa_db_host = config['configurations']['admin-properties']['db_host']
+ repo_name = str(config['clusterName']) + '_solr'
+
+ ranger_env = config['configurations']['ranger-env']
+ ranger_plugin_properties = config['configurations']['ranger-solr-plugin-properties']
+
+ ranger_solr_audit = config['configurations']['ranger-solr-audit']
+ ranger_solr_audit_attrs = config['configuration_attributes']['ranger-solr-audit']
+ ranger_solr_security = config['configurations']['ranger-solr-security']
+ ranger_solr_security_attrs = config['configuration_attributes']['ranger-solr-security']
+ ranger_solr_policymgr_ssl = config['configurations']['ranger-solr-policymgr-ssl']
+ ranger_solr_policymgr_ssl_attrs = config['configuration_attributes']['ranger-solr-policymgr-ssl']
+
+ policy_user = config['configurations']['ranger-solr-plugin-properties']['policy_user']
+
+ ranger_plugin_config = {
+ 'username' : config['configurations']['ranger-solr-plugin-properties']['REPOSITORY_CONFIG_USERNAME'],
+ 'password' : unicode(config['configurations']['ranger-solr-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']),
+ 'solr.url' : config['configurations']['ranger-solr-plugin-properties']['solr.url'],
+ 'commonNameForCertificate' : config['configurations']['ranger-solr-plugin-properties']['common.name.for.certificate']
+ }
+
+ solr_ranger_plugin_repo = {
+ 'isEnabled': 'true',
+ 'configs': ranger_plugin_config,
+ 'description': 'solr repo',
+ 'name': repo_name,
+ 'repositoryType': 'solr',
+ 'type': 'solr',
+ 'assetType': '1'
+ }
+
+ if stack_supports_ranger_kerberos and security_enabled:
+ ranger_plugin_config['policy.download.auth.users'] = solr_user
+ ranger_plugin_config['tag.download.auth.users'] = solr_user
+ ranger_plugin_config['ambari.service.check.user'] = policy_user
+
+ #For curl command in ranger plugin to get db connector
+ jdk_location = config['hostLevelParams']['jdk_location']
+ java_share_dir = '/usr/share/java'
+ previous_jdbc_jar_name = None
+
+ if stack_supports_ranger_audit_db:
+ if xa_audit_db_flavor and xa_audit_db_flavor == 'mysql':
+ jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
+ jdbc_driver = "com.mysql.jdbc.Driver"
+ elif xa_audit_db_flavor and xa_audit_db_flavor == 'oracle':
+ jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
+ colon_count = xa_db_host.count(':')
+ if colon_count == 2 or colon_count == 0:
+ audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
+ else:
+ audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
+ jdbc_driver = "oracle.jdbc.OracleDriver"
+ elif xa_audit_db_flavor and xa_audit_db_flavor == 'postgres':
+ jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}')
+ jdbc_driver = "org.postgresql.Driver"
+ elif xa_audit_db_flavor and xa_audit_db_flavor == 'mssql':
+ jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}')
+ jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
+ elif xa_audit_db_flavor and xa_audit_db_flavor == 'sqla':
+ jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
+ audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
+ jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
+
+ downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ driver_curl_target = format("{solr_home}/libs/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+ previous_jdbc_jar = format("{solr_home}/libs/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+
+ xa_audit_db_is_enabled = False
+ ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
+ if xml_configurations_supported and stack_supports_ranger_audit_db:
+ xa_audit_db_is_enabled = config['configurations']['ranger-solr-audit']['xasecure.audit.destination.db']
+ xa_audit_hdfs_is_enabled = default('/configurations/ranger-solr-audit/xasecure.audit.destination.hdfs', False)
+ ssl_keystore_password = unicode(config['configurations']['ranger-solr-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
+ ssl_truststore_password = unicode(config['configurations']['ranger-solr-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
+ credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
+
+ stack_version = get_stack_version('solr-server')
+ setup_ranger_env_sh_source = format('{stack_root}/{stack_version}/ranger-solr-plugin/install/conf.templates/enable/solr-ranger-env.sh')
+ setup_ranger_env_sh_target = format("{solr_conf}/solr-ranger-env.sh")
+
+ #For SQLA explicitly disable audit to DB for Ranger
+ if xa_audit_db_flavor == 'sqla':
+ xa_audit_db_is_enabled = False
+
+ namenode_hosts = default("/clusterHostInfo/namenode_host", [])
+ has_namenode = not len(namenode_hosts) == 0
+
+
+# *********************** end RANGER PLUGIN CHANGES ****************
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+kinit_path_local = status_params.kinit_path_local
+
+if 'ranger-env' in config['configurations']:
+ stack_root = Script.get_stack_root()
+ ranger_home = format('{stack_root}/current/ranger-admin')
+ audit_solr_enabled = default('/configurations/ranger-env/xasecure.audit.destination.solr', False)
+ ranger_solr_config_set = config['configurations']['ranger-env']['ranger_solr_config_set']
+ ranger_solr_collection_name = config['configurations']['ranger-env']['ranger_solr_collection_name']
+ ranger_solr_shards = config['configurations']['ranger-env']['ranger_solr_shards']
+ replication_factor = config['configurations']['ranger-env']['ranger_solr_replication_factor']
+ ranger_solr_conf = format('{solr_dir}/server/solr/configsets/ranger_audit_configs/conf')
+ is_solrCloud_enabled = default('/configurations/ranger-env/is_solrCloud_enabled', False)
+ is_external_solrCloud_enabled = default('/configurations/ranger-env/is_external_solrCloud_enabled', False)
+ stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
+
+
+import functools
+#create partial functions with common arguments for every HdfsDirectory call
+#to create hdfs directory we need to call params.HdfsDirectory in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs
+)
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/service_check.py
new file mode 100644
index 0000000..9fc458a
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/service_check.py
@@ -0,0 +1,48 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script.script import Script
+
+class ServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ Logger.info('Solr Service Check ...')
+ if "solr-env" in params.config['configurations'] \
+ and params.solr_hosts is not None \
+ and len(params.solr_hosts) > 0:
+ solr_protocol = "https" if params.solr_ssl_enabled else "http"
+ solr_host = params.solr_hosts[0] # choose the first solr host
+ solr_port = params.solr_port
+ solr_url = format("{solr_protocol}://{solr_host}:{solr_port}/solr/#/")
+
+ smokeuser_kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};") if params.security_enabled else ""
+ smoke_solr_cmd = format("{smokeuser_kinit_cmd} curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k {solr_url} | grep 200")
+ Execute(smoke_solr_cmd,
+ tries = 40,
+ try_sleep=3,
+ user=params.smokeuser,
+ logoutput=True)
+
+if __name__ == "__main__":
+ ServiceCheck().execute()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/setup_ranger_solr.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/setup_ranger_solr.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/setup_ranger_solr.py
new file mode 100644
index 0000000..e5f0b69
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/setup_ranger_solr.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources import File, Execute
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions.format import format
+from resource_management.core.resources.system import Directory
+
+
+def setup_ranger_solr():
+ import params
+
+ if params.has_ranger_admin and params.security_enabled:
+
+ from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
+
+ if params.retryAble:
+ Logger.info("Solr: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+ else:
+ Logger.info("Solr: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
+ if params.xml_configurations_supported and params.enable_ranger_solr and params.xa_audit_hdfs_is_enabled:
+ if params.has_namenode:
+ params.HdfsResource("/ranger/audit",
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ group=params.hdfs_user,
+ mode=0755,
+ recursive_chmod=True
+ )
+ params.HdfsResource("/ranger/audit/solr",
+ type="directory",
+ action="create_on_execute",
+ owner=params.solr_user,
+ group=params.solr_user,
+ mode=0700,
+ recursive_chmod=True
+ )
+ params.HdfsResource(None, action="execute")
+
+
+ File(os.path.join(params.stack_root, params.stack_version, "ranger-solr-plugin", "ranger_credential_helper.py"),
+ mode=0755)
+
+ jar_files = os.listdir(os.path.join(params.stack_root, params.stack_version, "ranger-solr-plugin/lib"))
+
+ for jar_file in jar_files:
+ plugin_dir = os.path.join(params.stack_root, params.stack_version, "ranger-solr-plugin/lib", jar_file)
+
+ Execute(('ln','-sf', plugin_dir,
+ os.path.join(params.stack_root, params.stack_version, "solr/server/solr-webapp/webapp/WEB-INF/lib", jar_file)),
+ only_if=format('ls {plugin_dir}'),
+ sudo=True)
+
+
+ setup_ranger_plugin('solr-server', 'solr', params.previous_jdbc_jar,
+ params.downloaded_custom_connector, params.driver_curl_source,
+ params.driver_curl_target, params.java64_home,
+ params.repo_name, params.solr_ranger_plugin_repo,
+ params.ranger_env, params.ranger_plugin_properties,
+ params.policy_user, params.policymgr_mgr_url,
+ params.enable_ranger_solr, conf_dict=params.solr_conf,
+ component_user=params.solr_user, component_group=params.user_group, cache_service_list=['solr'],
+ plugin_audit_properties=params.ranger_solr_audit, plugin_audit_attributes=params.ranger_solr_audit_attrs,
+ plugin_security_properties=params.ranger_solr_security, plugin_security_attributes=params.ranger_solr_security_attrs,
+ plugin_policymgr_ssl_properties=params.ranger_solr_policymgr_ssl, plugin_policymgr_ssl_attributes=params.ranger_solr_policymgr_ssl_attrs,
+ component_list=['solr'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
+ credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
+ ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
+ api_version = 'v2', skip_if_rangeradmin_down= not params.retryAble,
+ is_security_enabled = params.security_enabled,
+ is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
+ component_user_principal=params.solr_kerberos_principal if params.security_enabled else None,
+ component_user_keytab=params.solr_kerberos_keytab if params.security_enabled else None)
+
+ properties_files = os.listdir(format('/etc/solr/conf'))
+
+ if params.security_enabled and params.enable_ranger_solr:
+ solr_classes_dir = os.path.join(params.stack_root, params.stack_version, "solr/server/solr-webapp/webapp/WEB-INF/classes")
+ Directory(solr_classes_dir,
+ owner=params.solr_user,
+ group=params.user_group,
+ ignore_failures=True
+ )
+
+
+ Execute(format('mkdir {solr_classes_dir}'),
+ not_if=format('ls {solr_classes_dir}'))
+
+ for properties_file in properties_files:
+ Execute(('ln','-sf',format('/etc/solr/conf/{properties_file}'),
+ os.path.join(params.stack_root, params.stack_version, "solr/server/solr-webapp/webapp/WEB-INF/classes", properties_file)),
+ only_if=format('ls /etc/solr/conf/{properties_file}'),
+ sudo=True)
+
+ zk_port = ":" + params.zookeeper_port + ","
+
+ if params.enable_ranger_solr:
+ zookeeper_hosts_ip = zk_port.join(params.zookeeper_hosts_list) + ":" + params.zookeeper_port
+ zookeeper_script = os.path.join(params.stack_root, params.stack_version, "solr/server/scripts/cloud-scripts/zkcli.sh")
+
+ set_solr_ranger_authorizer = format('{zookeeper_script} -zkhost {zookeeper_hosts_ip} ' +
+ '-cmd put /solr/security.json \'{{\"authentication":{{\"class\":\"org.apache.solr.security.KerberosPlugin\"}},\"authorization\":{{\"class\": '+
+ '\"org.apache.ranger.authorization.solr.authorizer.RangerSolrAuthorizer\"}}}}\'')
+
+ Execute(set_solr_ranger_authorizer)
+
+ else:
+ Logger.info('Ranger admin not installed')
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/setup_solr.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/setup_solr.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/setup_solr.py
new file mode 100644
index 0000000..c20623f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/setup_solr.py
@@ -0,0 +1,129 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import Fail
+from resource_management.core.source import InlineTemplate, Template, StaticFile
+from resource_management.core.resources.system import Directory, File
+from resource_management.libraries.functions.decorator import retry
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions import solr_cloud_util
+from resource_management.libraries.functions.default import default
+
+def setup_solr(name = None):
+ import params
+
+ if name == 'server':
+ params.HdfsResource(params.solr_hdfs_home_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.solr_user,
+ mode=0755
+ )
+
+ params.HdfsResource(None, action="execute")
+
+ Directory([params.solr_log_dir, params.solr_piddir,
+ params.solr_datadir, params.solr_data_resources_dir],
+ mode=0755,
+ cd_access='a',
+ create_parents=True,
+ owner=params.solr_user,
+ group=params.user_group
+ )
+
+ Directory([params.solr_dir],
+ mode=0755,
+ cd_access='a',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ Directory([params.solr_conf],
+ mode=0755,
+ cd_access='a',
+ owner=params.solr_user,
+ group=params.user_group,
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ File(params.solr_log,
+ mode=0644,
+ owner=params.solr_user,
+ group=params.user_group,
+ content=''
+ )
+
+ File(format("{solr_conf}/solr-env.sh"),
+ content=InlineTemplate(params.solr_env_content),
+ mode=0755,
+ owner=params.solr_user,
+ group=params.user_group
+ )
+
+ if params.solr_xml_content:
+ File(format("{solr_datadir}/solr.xml"),
+ content=InlineTemplate(params.solr_xml_content),
+ owner=params.solr_user,
+ group=params.user_group
+ )
+
+ File(format("{solr_conf}/log4j.properties"),
+ content=InlineTemplate(params.solr_log4j_content),
+ owner=params.solr_user,
+ group=params.user_group
+ )
+
+ jaas_file = params.solr_jaas_file if params.security_enabled else None
+ url_scheme = 'https' if params.solr_ssl_enabled else 'http'
+
+ create_solr_znode()
+
+ if params.security_enabled:
+ File(format("{solr_jaas_file}"),
+ content=Template("solr_jaas.conf.j2"),
+ owner=params.solr_user)
+
+ solr_cloud_util.set_cluster_prop(
+ zookeeper_quorum=params.zookeeper_quorum,
+ solr_znode=params.solr_znode,
+ java64_home=params.java64_home,
+ prop_name="urlScheme",
+ prop_value=url_scheme,
+ jaas_file=jaas_file
+ )
+
+ solr_cloud_util.setup_kerberos_plugin(
+ zookeeper_quorum=params.zookeeper_quorum,
+ solr_znode=params.solr_znode,
+ jaas_file=jaas_file,
+ java64_home=params.java64_home,
+ secure=params.security_enabled
+ )
+
+ else :
+ raise Fail('Neither client, nor server were selected to install.')
+
+@retry(times=30, sleep_time=5, err_class=Fail)
+def create_solr_znode():
+ import params
+ solr_cloud_util.create_znode(
+ zookeeper_quorum=params.zookeeper_quorum,
+ solr_znode=params.solr_znode,
+ java64_home=params.java64_home)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/solr.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/solr.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/solr.py
new file mode 100644
index 0000000..7331cfd
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/solr.py
@@ -0,0 +1,161 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import sys
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.logger import Logger
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.libraries.functions.get_user_call_output import get_user_call_output
+from resource_management.libraries.functions.show_logs import show_logs
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.version import compare_versions
+from resource_management.libraries.functions.version import format_stack_version
+from setup_solr import setup_solr
+from setup_ranger_solr import setup_ranger_solr
+from resource_management.libraries.functions import solr_cloud_util
+
+
+class Solr(Script):
+ def install(self, env):
+ import params
+ env.set_params(params)
+ self.install_packages(env)
+
+ def configure(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ setup_solr(name = 'server')
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ if params.security_enabled:
+ solr_kinit_cmd = format("{kinit_path_local} -kt {solr_kerberos_keytab} {solr_kerberos_principal}; ")
+ Execute(solr_kinit_cmd, user=params.solr_user)
+
+ if params.is_supported_solr_ranger:
+ setup_ranger_solr() #Ranger Solr Plugin related call
+
+ if params.restart_during_downgrade:
+ solr_env = {'SOLR_INCLUDE': format('{solr_conf}/solr.in.sh')}
+ else:
+ solr_env = {'SOLR_INCLUDE': format('{solr_conf}/solr-env.sh')}
+ Execute(
+ format('{solr_bindir}/solr start -cloud -noprompt -s {solr_datadir} >> {solr_log} 2>&1'),
+ environment=solr_env,
+ user=params.solr_user
+ )
+
+ if 'ranger-env' in params.config['configurations'] and params.audit_solr_enabled:
+ solr_cloud_util.upload_configuration_to_zk(
+ zookeeper_quorum=params.zookeeper_quorum,
+ solr_znode=params.solr_znode,
+ config_set=params.ranger_solr_config_set,
+ config_set_dir=params.ranger_solr_conf,
+ tmp_dir=params.tmp_dir,
+ java64_home=params.java64_home,
+ jaas_file=params.solr_jaas_file,
+ retry=30, interval=5)
+
+ solr_cloud_util.create_collection(
+ zookeeper_quorum=params.zookeeper_quorum,
+ solr_znode=params.solr_znode,
+ collection=params.ranger_solr_collection_name,
+ config_set=params.ranger_solr_config_set,
+ java64_home=params.java64_home,
+ shards=params.ranger_solr_shards,
+ replication_factor=int(params.replication_factor),
+ jaas_file=params.solr_jaas_file)
+
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ try:
+ env = format('{solr_conf}/solr-env.sh')
+ if not os.path.exists(env):
+ old_env = format('{solr_conf}/solr.in.sh')
+ if os.path.exists(old_env):
+ env = old_env
+ else:
+ self.configure(env)
+
+ no_op_test = format("! ((`SOLR_INCLUDE={env} {solr_bindir}/solr status | grep process | wc -l`))")
+ Execute(format('{solr_bindir}/solr stop -all >> {solr_log}'),
+ environment={'SOLR_INCLUDE': env},
+ user=params.solr_user,
+ not_if=no_op_test
+ )
+
+ File(params.solr_pidfile,
+ action="delete"
+ )
+ except:
+ Logger.warning("Could not stop solr:" + str(sys.exc_info()[1]) + "\n Trying to kill it")
+ self.kill_process(params.solr_pidfile, params.solr_user, params.solr_log_dir)
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+
+ check_process_status(status_params.solr_pidfile)
+
+ def kill_process(self, pid_file, user, log_dir):
+ import params
+ """
+ Kill the process by pid file, then check the process is running or not. If the process is still running after the kill
+ command, it will try to kill with -9 option (hard kill)
+ """
+ pid = get_user_call_output(format("cat {pid_file}"), user=user, is_checked_call=False)[1]
+ process_id_exists_command = format("ls {pid_file} >/dev/null 2>&1 && ps -p {pid} >/dev/null 2>&1")
+
+ kill_cmd = format("{sudo} kill {pid}")
+ Execute(kill_cmd,
+ not_if=format("! ({process_id_exists_command})"))
+ wait_time = 5
+
+ hard_kill_cmd = format("{sudo} kill -9 {pid}")
+ Execute(hard_kill_cmd,
+ not_if=format("! ({process_id_exists_command}) || ( sleep {wait_time} && ! ({process_id_exists_command}) )"),
+ ignore_failures=True)
+ try:
+ Execute(format("! ({process_id_exists_command})"),
+ tries=20,
+ try_sleep=3,
+ )
+ except:
+ show_logs(log_dir, user)
+ raise
+
+ File(pid_file,
+ action="delete"
+ )
+
+
+if __name__ == "__main__":
+ Solr().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/solr_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/solr_client.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/solr_client.py
new file mode 100644
index 0000000..3a75e8d
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/solr_client.py
@@ -0,0 +1,51 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ClientComponentHasNoStatus
+from resource_management.libraries.script.script import Script
+from setup_solr import setup_solr
+
+class SolrClient(Script):
+
+ def install(self, env):
+ import params
+ env.set_params(params)
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ setup_solr(name ='client')
+
+ def start(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ def stop(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+
+if __name__ == "__main__":
+ SolrClient().execute()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/status_params.py
new file mode 100644
index 0000000..a740242
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/scripts/status_params.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script.script import Script
+
+config = Script.get_config()
+
+solr_port = default('configurations/solr-env/solr_port', '8983')
+solr_piddir = default('configurations/solr-env/solr_pid_dir', '/var/run/solr')
+solr_pidfile = format("{solr_piddir}/solr-{solr_port}.pid")
+
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/templates/solr_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/templates/solr_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/templates/solr_jaas.conf.j2
new file mode 100644
index 0000000..24df10f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/package/templates/solr_jaas.conf.j2
@@ -0,0 +1,26 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+Client {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ storeKey=true
+ useTicketCache=false
+ keyTab="{{solr_kerberos_keytab}}"
+ principal="{{solr_kerberos_principal}}";
+};
\ No newline at end of file
[07/23] ambari git commit: AMBARI-21473. Zeppelin does not start and
returns: params.zookeeper_znode_parent not in
interpreter['properties']['phoenix.url']: KeyError: 'phoenix.url'(Prabhjyot
Singh via Venkata Sairam)
Posted by jo...@apache.org.
AMBARI-21473. Zeppelin does not start and returns: params.zookeeper_znode_parent not in interpreter['properties']['phoenix.url']: KeyError: 'phoenix.url'(Prabhjyot Singh via Venkata Sairam)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/837b0bdf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/837b0bdf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/837b0bdf
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 837b0bdff6cc38c7c2ffe548bb012ac84b5bca7b
Parents: 9ac42d0
Author: Venkata Sairam <ve...@gmail.com>
Authored: Wed Jul 19 14:27:31 2017 +0530
Committer: Venkata Sairam <ve...@gmail.com>
Committed: Wed Jul 19 14:39:10 2017 +0530
----------------------------------------------------------------------
.../common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/837b0bdf/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index 0013ab0..ba46dc8 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -295,6 +295,7 @@ class Master(Script):
interpreter['properties']['zeppelin.jdbc.keytab.location'] = params.zeppelin_kerberos_keytab
if params.zookeeper_znode_parent \
and params.hbase_zookeeper_quorum \
+ and 'phoenix.url' in interpreter['properties'] \
and params.zookeeper_znode_parent not in interpreter['properties']['phoenix.url']:
interpreter['properties']['phoenix.url'] = "jdbc:phoenix:" + \
params.hbase_zookeeper_quorum + ':' + \
[23/23] ambari git commit: Merge branch 'branch-2.5' into
branch-feature-AMBARI-21450
Posted by jo...@apache.org.
Merge branch 'branch-2.5' into branch-feature-AMBARI-21450
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/23db89b4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/23db89b4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/23db89b4
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 23db89b4a9cb40343318d029722b95392c255062
Parents: 81035d8 212ee1c
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Fri Jul 21 00:37:41 2017 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Fri Jul 21 00:37:41 2017 -0400
----------------------------------------------------------------------
.../resource_management/TestUserResource.py | 2 +-
.../python/resource_management/core/base.py | 11 +
.../core/providers/accounts.py | 6 +-
.../core/resources/accounts.py | 4 +-
.../libraries/functions/stack_features.py | 2 +-
.../actionmanager/ExecutionCommandWrapper.java | 34 +-
.../AmbariCustomCommandExecutionHelper.java | 13 +-
.../AmbariManagementControllerImpl.java | 28 +-
.../internal/AbstractProviderModule.java | 1 +
.../internal/ClientConfigResourceProvider.java | 3 +-
.../internal/HttpPropertyProvider.java | 27 +-
.../internal/UpgradeResourceProvider.java | 91 +++--
.../listeners/upgrade/StackVersionListener.java | 33 +-
.../ambari/server/state/ConfigHelper.java | 33 +-
.../0.1.0.2.3/package/scripts/atlas_client.py | 2 +-
.../ATLAS/0.1.0.2.3/package/scripts/params.py | 2 +
.../HDFS/2.1.0.2.0/package/scripts/namenode.py | 4 +
.../2.1.0.2.0/package/scripts/zkfc_slave.py | 8 +-
.../2.1.0.2.0/package/scripts/historyserver.py | 2 +-
.../2.1.0.2.0/package/scripts/params_linux.py | 11 +-
.../YARN/2.1.0.2.0/package/scripts/service.py | 4 +
.../2.1.0.2.0/package/scripts/service_check.py | 6 +-
.../0.6.0.2.5/package/scripts/master.py | 1 +
.../BigInsights/4.2.5/services/SOLR/alerts.json | 37 +++
.../SOLR/configuration/ranger-solr-audit.xml | 127 +++++++
.../ranger-solr-plugin-properties.xml | 88 +++++
.../configuration/ranger-solr-policymgr-ssl.xml | 66 ++++
.../SOLR/configuration/ranger-solr-security.xml | 58 ++++
.../SOLR/configuration/solr-client-log4j.xml | 48 +++
.../services/SOLR/configuration/solr-env.xml | 263 +++++++++++++++
.../services/SOLR/configuration/solr-log4j.xml | 36 ++
.../services/SOLR/configuration/solr-xml.xml | 36 ++
.../4.2.5/services/SOLR/kerberos.json | 51 +++
.../4.2.5/services/SOLR/metainfo.xml | 124 +++++++
.../services/SOLR/package/scripts/params.py | 331 +++++++++++++++++++
.../SOLR/package/scripts/service_check.py | 48 +++
.../SOLR/package/scripts/setup_ranger_solr.py | 126 +++++++
.../services/SOLR/package/scripts/setup_solr.py | 129 ++++++++
.../4.2.5/services/SOLR/package/scripts/solr.py | 161 +++++++++
.../SOLR/package/scripts/solr_client.py | 51 +++
.../SOLR/package/scripts/status_params.py | 34 ++
.../SOLR/package/templates/solr_jaas.conf.j2 | 26 ++
.../properties/solr-client-log4j.properties.j2 | 38 +++
.../services/SOLR/properties/solr-env.sh.j2 | 126 +++++++
.../SOLR/properties/solr-log4j.properties.j2 | 40 +++
.../4.2.5/services/SOLR/properties/solr.xml.j2 | 26 ++
.../services/SOLR/quicklinks/quicklinks.json | 37 +++
.../4.2.5/services/SOLR/themes/theme.json | 107 ++++++
.../4.2.5/upgrades/config-upgrade.xml | 7 +
.../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 5 +
.../HBASE/package/scripts/hbase_upgrade.py | 2 +-
.../BigInsights/4.2/upgrades/config-upgrade.xml | 7 +
.../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 5 +
.../hooks/before-ANY/files/changeToSecureUid.sh | 2 +
.../before-ANY/scripts/shared_initialization.py | 27 +-
.../stacks/HDP/2.5/services/stack_advisor.py | 7 +-
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 2 +-
.../src/main/resources/stacks/stack_advisor.py | 7 +-
.../ExecutionCommandWrapperTest.java | 78 ++++-
.../AmbariManagementControllerImplTest.java | 3 +-
.../internal/HttpPropertyProviderTest.java | 11 +
.../upgrade/StackVersionListenerTest.java | 60 ++--
.../src/test/python/TestStackFeature.py | 61 ++++
.../2.0.6/hooks/before-ANY/test_before_any.py | 45 ++-
.../stacks/2.5/common/test_stack_advisor.py | 4 +-
ambari-web/app/mappers/hosts_mapper.js | 2 +-
ambari-web/app/messages.js | 2 +-
.../stack_upgrade/upgrade_version_box_view.js | 31 +-
.../upgrade_version_column_view.js | 38 ---
.../upgrade_version_box_view_test.js | 43 +++
70 files changed, 2795 insertions(+), 196 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/23db89b4/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
index efd609a,41521f8..91db7d0
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
@@@ -17,8 -17,13 +17,11 @@@
*/
package org.apache.ambari.server.actionmanager;
+ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
+ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
+
-import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
-import java.util.Set;
import java.util.TreeMap;
import org.apache.ambari.server.AmbariException;
@@@ -27,15 -32,16 +30,19 @@@ import org.apache.ambari.server.Service
import org.apache.ambari.server.agent.AgentCommand.AgentCommandType;
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.agent.ExecutionCommand.KeyNames;
+ import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
-import org.apache.ambari.server.orm.entities.ClusterVersionEntity;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.DesiredConfig;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;
+ import org.apache.ambari.server.state.ServiceInfo;
+ import org.apache.ambari.server.state.StackId;
+ import org.apache.ambari.server.state.StackInfo;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -186,44 -198,50 +199,63 @@@ public class ExecutionCommandWrapper
}
}
- Map<String,String> commandParams = executionCommand.getCommandParams();
-
- // set the version for the command if it's not already set
- if (!commandParams.containsKey(KeyNames.VERSION)) {
- // the cluster's effective version should be used for this command
- ClusterVersionEntity effectiveClusterVersion = cluster.getEffectiveClusterVersion();
-
- // in the event that the effective version is NULL (meaning that most
- // likely the cluster is still being provisioned), then send down the
- // version if this is not an install command
- if (null == effectiveClusterVersion
- && executionCommand.getRoleCommand() != RoleCommand.INSTALL) {
- Collection<ClusterVersionEntity> clusterVersions = cluster.getAllClusterVersions();
- if (clusterVersions.size() == 1) {
- effectiveClusterVersion = clusterVersions.iterator().next();
+ // set the repository version for the component this command is for -
+ // always use the current desired version
+ try {
+ RepositoryVersionEntity repositoryVersion = null;
+ String serviceName = executionCommand.getServiceName();
+ if (!StringUtils.isEmpty(serviceName)) {
+ Service service = cluster.getService(serviceName);
+ if (null != service) {
+ repositoryVersion = service.getDesiredRepositoryVersion();
}
- }
- if (null != effectiveClusterVersion) {
- commandParams.put(KeyNames.VERSION,
- effectiveClusterVersion.getRepositoryVersion().getVersion());
+ String componentName = executionCommand.getComponentName();
+ if (!StringUtils.isEmpty(componentName)) {
+ ServiceComponent serviceComponent = service.getServiceComponent(
+ executionCommand.getComponentName());
+
+ if (null != serviceComponent) {
+ repositoryVersion = serviceComponent.getDesiredRepositoryVersion();
+ }
+ }
}
- }
- // add the stack and common-services folders to the command, but only if
- // they don't exist - they may have been put on here with specific
- // values ahead of time
- StackId stackId = cluster.getDesiredStackVersion();
- StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(),
- stackId.getStackVersion());
++ Map<String, String> commandParams = executionCommand.getCommandParams();
+
- if (!commandParams.containsKey(HOOKS_FOLDER)) {
- commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
- }
+ if (null != repositoryVersion) {
- executionCommand.getCommandParams().put(KeyNames.VERSION, repositoryVersion.getVersion());
++ commandParams.put(KeyNames.VERSION, repositoryVersion.getVersion());
+ executionCommand.getHostLevelParams().put(KeyNames.CURRENT_VERSION, repositoryVersion.getVersion());
+
- if (!commandParams.containsKey(SERVICE_PACKAGE_FOLDER)) {
- String serviceName = executionCommand.getServiceName();
- if (!StringUtils.isEmpty(serviceName)) {
- ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(),
- stackId.getStackVersion(), serviceName);
++ StackId stackId = repositoryVersion.getStackId();
++ StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(),
++ stackId.getStackVersion());
++
++ if (!commandParams.containsKey(HOOKS_FOLDER)) {
++ commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
++ }
++
++ if (!commandParams.containsKey(SERVICE_PACKAGE_FOLDER)) {
++ if (!StringUtils.isEmpty(serviceName)) {
++ ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(),
++ stackId.getStackVersion(), serviceName);
+
- commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
++ commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
++ }
++ }
}
+ } catch (ServiceNotFoundException serviceNotFoundException) {
+ // it's possible that there are commands specified for a service where
+ // the service doesn't exist yet
+ LOG.warn(
+ "The service {} is not installed in the cluster. No repository version will be sent for this command.",
+ executionCommand.getServiceName());
}
+
+ // set the desired versions of versionable components. This is safe even during an upgrade because
+ // we are "loading-late": components that have not yet upgraded in an EU will have the correct versions.
+ executionCommand.setComponentVersions(cluster);
+
} catch (ClusterNotFoundException cnfe) {
// it's possible that there are commands without clusters; in such cases,
// just return the de-serialized command and don't try to read configs
http://git-wip-us.apache.org/repos/asf/ambari/blob/23db89b4/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index 011ebfd,723a10d..06833c4
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@@ -92,8 -88,10 +90,9 @@@ import org.apache.ambari.server.state.H
import org.apache.ambari.server.state.HostComponentAdminState;
import org.apache.ambari.server.state.HostState;
import org.apache.ambari.server.state.MaintenanceState;
+ import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.PropertyInfo.PropertyType;
import org.apache.ambari.server.state.RepositoryInfo;
-import org.apache.ambari.server.state.RepositoryVersionState;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.state.ServiceComponentHost;
@@@ -477,22 -466,7 +477,20 @@@ public class AmbariCustomCommandExecuti
commandTimeout = Math.max(60, commandTimeout);
}
+ if (requestParams != null && requestParams.containsKey("context")) {
+ String requestContext = requestParams.get("context");
+ if (StringUtils.isNotEmpty(requestContext) && requestContext.toLowerCase().contains("rolling-restart")) {
+ Config clusterEnvConfig = cluster.getDesiredConfigByType("cluster-env");
+ if (clusterEnvConfig != null) {
+ String componentRollingRestartTimeout = clusterEnvConfig.getProperties().get("namenode_rolling_restart_timeout");
+ if (StringUtils.isNotEmpty(componentRollingRestartTimeout)) {
+ commandTimeout = Integer.parseInt(componentRollingRestartTimeout);
+ }
+ }
+ }
+ }
+
commandParams.put(COMMAND_TIMEOUT, "" + commandTimeout);
- commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
- commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
Map<String, String> roleParams = execCmd.getRoleParams();
if (roleParams == null) {
@@@ -804,15 -770,6 +802,13 @@@
}
commandParams.put(COMMAND_TIMEOUT, commandTimeout);
+ String checkType = configHelper.getValueFromDesiredConfigurations(cluster, ConfigHelper.CLUSTER_ENV, ConfigHelper.SERVICE_CHECK_TYPE);
+ if (ConfigHelper.SERVICE_CHECK_MINIMAL.equals(checkType)) {
+ int actualTimeout = Integer.parseInt(commandParams.get(COMMAND_TIMEOUT)) / 2;
+ actualTimeout = actualTimeout < MIN_STRICT_SERVICE_CHECK_TIMEOUT ? MIN_STRICT_SERVICE_CHECK_TIMEOUT : actualTimeout;
+ commandParams.put(COMMAND_TIMEOUT, Integer.toString(actualTimeout));
+ }
- commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
- commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
+
execCmd.setCommandParams(commandParams);
if (actionParameters != null) { // If defined
http://git-wip-us.apache.org/repos/asf/ambari/blob/23db89b4/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index ad4d322,850838d..becf596
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@@ -28,7 -28,7 +28,6 @@@ import static org.apache.ambari.server.
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_FOLDER;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
--import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MAX_DURATION_OF_RETRIES;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST;
@@@ -36,7 -36,7 +35,6 @@@ import static org.apache.ambari.server.
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
--import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.UNLIMITED_KEY_JCE_REQUIRED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_GROUPS;
@@@ -1541,7 -1559,7 +1539,7 @@@ public class AmbariManagementController
Map.Entry<String, String> pair = (Map.Entry) it.next();
// Check the value if both keys exist
if (newConfigValues.containsKey(pair.getKey())) {
-- if (!newConfigValues.get((String) pair.getKey()).equals(pair.getValue())) {
++ if (!newConfigValues.get(pair.getKey()).equals(pair.getValue())) {
configsChanged.put(pair.getKey(), "changed");
}
} else {
@@@ -2447,9 -2462,9 +2445,6 @@@
}
commandParams.put(COMMAND_TIMEOUT, actualTimeout);
-- commandParams.put(SERVICE_PACKAGE_FOLDER,
-- serviceInfo.getServicePackageFolder());
-- commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
String customCacheDirectory = componentInfo.getCustomFolder();
if (customCacheDirectory != null) {
@@@ -2697,6 -2727,13 +2693,9 @@@
return requestStages;
}
+ // check all stack configs are present in desired configs
+ configHelper.checkAllStageConfigsPresentInDesiredConfigs(cluster);
+
-
- // caching effective cluster version
- ClusterVersionEntity effectiveClusterVersion = cluster.getEffectiveClusterVersion();
-
// caching upgrade suspended
boolean isUpgradeSuspended = cluster.isUpgradeSuspended();
http://git-wip-us.apache.org/repos/asf/ambari/blob/23db89b4/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index a712d82,de3225f..c26f062
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@@ -873,44 -1404,23 +930,34 @@@ public class UpgradeResourceProvider ex
}
Cluster cluster = context.getCluster();
- StackId effectiveStackId = effectiveRepositoryVersion.getStackId();
-
- // add each host to this stage
- RequestResourceFilter filter = new RequestResourceFilter("", "",
- new ArrayList<>(wrapper.getHosts()));
LOG.debug("Analyzing upgrade item {} with tasks: {}.", entity.getText(), entity.getTasks());
- Map<String, String> params = getNewParameterMap(request, context);
- params.put(UpgradeContext.COMMAND_PARAM_TASKS, entity.getTasks());
-
- // Apply additional parameters to the command that come from the stage.
- applyAdditionalParameters(wrapper, params);
-
- // Because custom task may end up calling a script/function inside a
- // service, it is necessary to set the
- // service_package_folder and hooks_folder params.
- AmbariMetaInfo ambariMetaInfo = s_metaProvider.get();
- StackId stackId = effectiveRepositoryVersion.getStackId();
-
- StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(),
- stackId.getStackVersion());
- Map<String, String> params = getNewParameterMap(request, context, effectiveRepositoryVersion);
+
+ // if the service/component are specified, then make sure to grab them off
+ // of the wrapper so they can be stored on the command for use later
+ String serviceName = null;
+ String componentName = null;
-
+ if (wrapper.getTasks() != null && wrapper.getTasks().size() > 0
+ && wrapper.getTasks().get(0).getService() != null) {
+ TaskWrapper taskWrapper = wrapper.getTasks().get(0);
+ serviceName = taskWrapper.getService();
+ componentName = taskWrapper.getComponent();
++ }
+
- ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(),
- stackId.getStackVersion(), serviceName);
++ Map<String, String> params = getNewParameterMap(request, context);
+ params.put(UpgradeContext.COMMAND_PARAM_TASKS, entity.getTasks());
- params.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
- params.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
- }
+ // Apply additional parameters to the command that come from the stage.
+ applyAdditionalParameters(wrapper, params);
+
+ // the ru_execute_tasks invokes scripts - it needs information about where
+ // the scripts live and for that it should always use the target repository
+ // stack
- applyRepositoryAssociatedParameters(wrapper, effectiveStackId, params);
++ applyRepositoryAssociatedParameters(wrapper, effectiveRepositoryVersion.getStackId(), params);
+
+ // add each host to this stage
+ RequestResourceFilter filter = new RequestResourceFilter(serviceName, componentName,
+ new ArrayList<>(wrapper.getHosts()));
ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
EXECUTE_TASK_ROLE, Collections.singletonList(filter), params);
@@@ -1054,6 -1584,9 +1101,10 @@@
// Apply additional parameters to the command that come from the stage.
applyAdditionalParameters(wrapper, commandParams);
+ // add things like hooks and service folders based on effective repo
- applyRepositoryAssociatedParameters(wrapper, effectiveStackId, commandParams);
++ applyRepositoryAssociatedParameters(wrapper, effectiveRepositoryVersion.getStackId(),
++ commandParams);
+
ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
"SERVICE_CHECK", filters, commandParams);
http://git-wip-us.apache.org/repos/asf/ambari/blob/23db89b4/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
index f8d5a5f,4183b51..b812476
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
@@@ -17,15 -17,23 +17,18 @@@
*/
package org.apache.ambari.server.events.listeners.upgrade;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.EagerSingleton;
+ import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent;
import org.apache.ambari.server.events.publishers.VersionEventPublisher;
import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
-import org.apache.ambari.server.orm.entities.ClusterVersionEntity;
-import org.apache.ambari.server.orm.entities.HostVersionEntity;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.Cluster;
+ import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.state.ServiceComponentHost;
+ import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.State;
import org.apache.ambari.server.state.UpgradeState;
import org.apache.commons.lang.StringUtils;
@@@ -93,24 -115,40 +104,34 @@@ public class StackVersionListener
}
}
- m_stackVersionLock.lock();
-
// Update host component version value if needed
try {
- ServiceComponent sc = cluster.getService(sch.getServiceName()).getServiceComponent(
- sch.getServiceComponentName());
+ // get the component information for the desired stack; if a component
+ // moves from UNKNOWN to providing a version, we must do the version
+ // advertised check against the target stack
- StackId desiredStackId = sch.getDesiredStackVersion();
++ StackId desiredStackId = sch.getDesiredStackId();
+
+ AmbariMetaInfo ambariMetaInfo = ambariMetaInfoProvider.get();
+ ComponentInfo componentInfo = ambariMetaInfo.getComponent(desiredStackId.getStackName(),
+ desiredStackId.getStackVersion(), sch.getServiceName(), sch.getServiceComponentName());
// not advertising a version, do nothing
- if (!sc.isVersionAdvertised()) {
+ if (!componentInfo.isVersionAdvertised()) {
// that's odd; a version came back - log it and still do nothing
if (!StringUtils.equalsIgnoreCase(UNKNOWN_VERSION, newVersion)) {
- LOG.debug(
+ LOG.warn(
"ServiceComponent {} doesn't advertise version, however ServiceHostComponent {} on host {} advertised version as {}. Skipping version update",
- sc.getName(), sch.getServiceComponentName(), sch.getHostName(), newVersion);
+ sch.getServiceComponentName(), sch.getServiceComponentName(), sch.getHostName(),
+ newVersion);
}
return;
}
+ ServiceComponent sc = cluster.getService(sch.getServiceName()).getServiceComponent(
+ sch.getServiceComponentName());
+
- boolean desiredVersionIsCurrentlyUnknown = StringUtils.equalsIgnoreCase(UNKNOWN_VERSION,
- sc.getDesiredVersion());
-
- // proces the UNKNOWN version being received or currently desired
- if (StringUtils.equalsIgnoreCase(UNKNOWN_VERSION, newVersion)
- || desiredVersionIsCurrentlyUnknown) {
+ // proces the UNKNOWN version
+ if (StringUtils.equalsIgnoreCase(UNKNOWN_VERSION, newVersion)) {
processUnknownDesiredVersion(cluster, sc, sch, newVersion);
return;
}
@@@ -213,6 -279,24 +234,6 @@@
// if the upgrade state changes, then also recalculate host versions
sch.setUpgradeState(upgradeState);
- }
-
- /**
- * Recalculates the {@link HostVersionEntity} for the host specified by the
- * host component, taking into account all component states on that host. This
- * will also trigger a {@link ClusterVersionEntity} recalculatation for the
- * cluster version as well.
- *
- * @param cluster
- * @param sch
- * @throws AmbariException
- */
- private void recalculateHostVersionAndClusterVersion(Cluster cluster, ServiceComponentHost sch)
- throws AmbariException {
- // trigger a re-calculation of the cluster state based on the SCH state
- RepositoryVersionEntity repoVersion = sch.recalculateHostVersionState();
- if (null != repoVersion) {
- cluster.recalculateClusterVersionState(repoVersion);
- }
+ sch.recalculateHostVersionState();
}
- }
+ }
http://git-wip-us.apache.org/repos/asf/ambari/blob/23db89b4/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index cd717c5,6e60890..974ad4f
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@@ -219,7 -219,8 +219,7 @@@ public class ConfigHelper
return resolved;
}
-
+
-
public Set<String> filterInvalidPropertyValues(Map<PropertyInfo, String> properties, String filteredListName) {
Set<String> resultSet = new HashSet<>();
for (Iterator<Entry<PropertyInfo, String>> iterator = properties.entrySet().iterator(); iterator.hasNext();) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/23db89b4/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
----------------------------------------------------------------------
diff --cc ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
index 64a1e3a,25c3381..4100f15
--- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
@@@ -273,6 -276,72 +280,69 @@@ public class ExecutionCommandWrapperTes
Assert.assertEquals(SERVICE_SITE_VAL6_H, mergedConfig.get(SERVICE_SITE_NAME6));
}
+ /**
+ * Test that the execution command wrapper properly sets the version
+ * information when the cluster is in the INSTALLING state.
+ *
+ * @throws JSONException
+ * @throws AmbariException
+ */
+ @Test
+ public void testExecutionCommandHasVersionInfoWithoutCurrentClusterVersion()
+ throws JSONException, AmbariException {
+ Cluster cluster = clusters.getCluster(CLUSTER1);
+
+ StackId stackId = cluster.getDesiredStackVersion();
+ RepositoryVersionEntity repositoryVersion = ormTestHelper.getOrCreateRepositoryVersion(stackId, "0.1-0000");
+
- cluster.createClusterVersion(stackId, repositoryVersion.getVersion(), "admin",
- RepositoryVersionState.INSTALLING);
-
+ // first try with an INSTALL command - this should not populate version info
+ ExecutionCommand executionCommand = new ExecutionCommand();
+ Map<String, String> commandParams = new HashMap<>();
+
+ executionCommand.setClusterName(CLUSTER1);
+ executionCommand.setTaskId(1);
+ executionCommand.setRequestAndStage(1, 1);
+ executionCommand.setHostname(HOST1);
+ executionCommand.setRole("NAMENODE");
+ executionCommand.setRoleParams(Collections.<String, String>emptyMap());
+ executionCommand.setRoleCommand(RoleCommand.INSTALL);
+ executionCommand.setServiceName("HDFS");
+ executionCommand.setCommandType(AgentCommandType.EXECUTION_COMMAND);
+ executionCommand.setCommandParams(commandParams);
+
+ String json = StageUtils.getGson().toJson(executionCommand, ExecutionCommand.class);
+ ExecutionCommandWrapper execCommWrap = new ExecutionCommandWrapper(json);
+ injector.injectMembers(execCommWrap);
+
+ ExecutionCommand processedExecutionCommand = execCommWrap.getExecutionCommand();
+ commandParams = processedExecutionCommand.getCommandParams();
+ Assert.assertFalse(commandParams.containsKey(KeyNames.VERSION));
+
+ // now try with a START command which should populate the version even
+ // though the state is INSTALLING
+ executionCommand = new ExecutionCommand();
+ commandParams = new HashMap<>();
+
+ executionCommand.setClusterName(CLUSTER1);
+ executionCommand.setTaskId(1);
+ executionCommand.setRequestAndStage(1, 1);
+ executionCommand.setHostname(HOST1);
+ executionCommand.setRole("NAMENODE");
+ executionCommand.setRoleParams(Collections.<String, String> emptyMap());
+ executionCommand.setRoleCommand(RoleCommand.START);
+ executionCommand.setServiceName("HDFS");
+ executionCommand.setCommandType(AgentCommandType.EXECUTION_COMMAND);
+ executionCommand.setCommandParams(commandParams);
+
+ json = StageUtils.getGson().toJson(executionCommand, ExecutionCommand.class);
+ execCommWrap = new ExecutionCommandWrapper(json);
+ injector.injectMembers(execCommWrap);
+
+ processedExecutionCommand = execCommWrap.getExecutionCommand();
+ commandParams = processedExecutionCommand.getCommandParams();
+ Assert.assertEquals("0.1-0000", commandParams.get(KeyNames.VERSION));
+ }
+
@AfterClass
public static void tearDown() throws AmbariException, SQLException {
H2DatabaseCleaner.clearDatabaseAndStopPersistenceService(injector);
http://git-wip-us.apache.org/repos/asf/ambari/blob/23db89b4/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
----------------------------------------------------------------------
diff --cc ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
index 5b4987e,35aa37f..4024f05
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
@@@ -2402,8 -2406,7 +2402,7 @@@ public class AmbariManagementController
controller.createCluster(cr);
// verification
- verify(injector, clusters, ambariMetaInfo, stackInfo, cluster, repoVersionDAO, repoVersion, clusterVersionDAO);
+ verify(injector, clusters, ambariMetaInfo, stackInfo, cluster, repoVersionDAO, repoVersion);
}
-
- }
+ }
http://git-wip-us.apache.org/repos/asf/ambari/blob/23db89b4/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
----------------------------------------------------------------------
diff --cc ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
index efe594a,3a4f5b9..cd8b499
--- a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
@@@ -63,18 -65,16 +63,17 @@@ public class StackVersionListenerTest e
private static final String UNKNOWN_VERSION = "UNKNOWN";
private static final String VALID_PREVIOUS_VERSION = "2.2.0.0";
private static final RepositoryVersionEntity DUMMY_REPOSITORY_VERSION_ENTITY = new RepositoryVersionEntity();
+ private static final HostVersionEntity DUMMY_HOST_VERSION_ENTITY = new HostVersionEntity();
private static final UpgradeEntity DUMMY_UPGRADE_ENTITY = new UpgradeEntity();
-- public static final String STACK_NAME = "HDP-2.4.0.0";
-- public static final String STACK_VERSION = "2.4.0.0";
++ public static final String STACK_NAME = "HDP";
++ public static final String STACK_VERSION = "2.4";
private Cluster cluster;
private ServiceComponentHost sch;
private Service service;
private ServiceComponent serviceComponent;
private VersionEventPublisher publisher = new VersionEventPublisher();
- private ComponentInfo componentInfo;
-- private StackId stackId;
++ private StackId stackId = new StackId(STACK_NAME, STACK_VERSION);
@TestSubject
private StackVersionListener listener = new StackVersionListener(publisher);
@@@ -88,21 -94,32 +93,28 @@@
sch = createNiceMock(ServiceComponentHost.class);
service = createNiceMock(Service.class);
serviceComponent = createNiceMock(ServiceComponent.class);
- stackId = createNiceMock(StackId.class);
+ componentInfo = createNiceMock(ComponentInfo.class);
- stackId = createNiceMock(StackId.class);
+
+ expect(cluster.getClusterId()).andReturn(CLUSTER_ID);
- expect(cluster.getService(SERVICE_NAME)).andReturn(service).atLeastOnce();
- expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(serviceComponent).atLeastOnce();
+ expect(cluster.getService(SERVICE_NAME)).andReturn(service).anyTimes();
- expect(cluster.getDesiredStackVersion()).andReturn(stackId).anyTimes();
- expect(stackId.getStackName()).andReturn(STACK_NAME).anyTimes();
- expect(stackId.getStackVersion()).andReturn(STACK_VERSION).anyTimes();
- expect(cluster.getClusterId()).andReturn(CLUSTER_ID).atLeastOnce();
-
- expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(serviceComponent).anyTimes();
- expect(serviceComponent.getDesiredStackVersion()).andReturn(stackId).anyTimes();
- expect(sch.getDesiredStackVersion()).andReturn(stackId).atLeastOnce();
++ expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(
++ serviceComponent).anyTimes();
++ expect(sch.getDesiredStackId()).andReturn(stackId).atLeastOnce();
expect(sch.getServiceName()).andReturn(SERVICE_NAME).atLeastOnce();
expect(sch.getServiceComponentName()).andReturn(SERVICE_COMPONENT_NAME).atLeastOnce();
+
+ expect(ambariMetaInfoProvider.get()).andReturn(ambariMetaInfo).atLeastOnce();
+ expect(ambariMetaInfo.getComponent(STACK_NAME, STACK_VERSION, SERVICE_NAME,
+ SERVICE_COMPONENT_NAME)).andReturn(componentInfo).atLeastOnce();
+
+ injectMocks(listener);
}
@Test
public void testRecalculateHostVersionStateWhenVersionIsNullAndNewVersionIsNotBlank() throws AmbariException {
- expect(sch.getVersion()).andReturn(null).atLeastOnce();
- expect(serviceComponent.getDesiredVersion()).andReturn(INVALID_NEW_VERSION).atLeastOnce();
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
+ expect(sch.getVersion()).andReturn(null);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setVersion(INVALID_NEW_VERSION);
expectLastCall().once();
expect(sch.recalculateHostVersionState()).andReturn(null).once();
@@@ -114,8 -131,8 +126,8 @@@
@Test
public void testRecalculateHostVersionStateWhenVersionIsUnknownAndNewVersionIsNotBlank() throws AmbariException {
- expect(sch.getVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce();
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
+ expect(sch.getVersion()).andReturn(UNKNOWN_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setVersion(INVALID_NEW_VERSION);
expectLastCall().once();
expect(sch.recalculateHostVersionState()).andReturn(null).once();
@@@ -127,11 -144,13 +139,11 @@@
@Test
public void testRecalculateClusterVersionStateWhenVersionIsNullAndNewVersionIsValid() throws AmbariException {
- expect(sch.getVersion()).andReturn(null).atLeastOnce();
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
+ expect(sch.getVersion()).andReturn(null);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setVersion(VALID_NEW_VERSION);
expectLastCall().once();
- expect(sch.recalculateHostVersionState()).andReturn(DUMMY_REPOSITORY_VERSION_ENTITY).once();
- cluster.recalculateClusterVersionState(DUMMY_REPOSITORY_VERSION_ENTITY);
- expectLastCall().once();
+ expect(sch.recalculateHostVersionState()).andReturn(DUMMY_HOST_VERSION_ENTITY).once();
replayAll();
@@@ -140,11 -159,13 +152,11 @@@
@Test
public void testRecalculateClusterVersionStateWhenVersionIsUnknownAndNewVersionIsValid() throws AmbariException {
- expect(sch.getVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce();
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
+ expect(sch.getVersion()).andReturn(UNKNOWN_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setVersion(VALID_NEW_VERSION);
expectLastCall().once();
- expect(sch.recalculateHostVersionState()).andReturn(DUMMY_REPOSITORY_VERSION_ENTITY).once();
- cluster.recalculateClusterVersionState(DUMMY_REPOSITORY_VERSION_ENTITY);
- expectLastCall().once();
+ expect(sch.recalculateHostVersionState()).andReturn(DUMMY_HOST_VERSION_ENTITY).once();
replayAll();
@@@ -153,8 -174,10 +165,8 @@@
@Test
public void testRecalculateHostVersionStateWhenComponentDesiredVersionIsUnknownAndNewVersionIsNotValid() throws AmbariException {
- expect(serviceComponent.getDesiredVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce();
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
- serviceComponent.setDesiredVersion(INVALID_NEW_VERSION);
- expectLastCall().once();
+ expect(serviceComponent.getDesiredVersion()).andReturn(UNKNOWN_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.NONE);
expectLastCall().once();
sch.setVersion(INVALID_NEW_VERSION);
@@@ -168,8 -191,10 +180,8 @@@
@Test
public void testRecalculateClusterVersionStateWhenComponentDesiredVersionIsUnknownAndNewVersionIsValid() throws AmbariException {
- expect(serviceComponent.getDesiredVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce();
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
- serviceComponent.setDesiredVersion(VALID_NEW_VERSION);
- expectLastCall().once();
+ expect(serviceComponent.getDesiredVersion()).andReturn(UNKNOWN_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.NONE);
expectLastCall().once();
sch.setVersion(VALID_NEW_VERSION);
@@@ -183,7 -210,7 +195,7 @@@
@Test
public void testRecalculateClusterVersionStateWhenVersionNotAdvertised() throws AmbariException {
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.FALSE);
- expect(componentInfo.isVersionAdvertised()).andReturn(false).atLeastOnce();
++ expect(componentInfo.isVersionAdvertised()).andReturn(false).once();
replayAll();
sendEventAndVerify(VALID_NEW_VERSION);
}
@@@ -191,7 -218,7 +203,7 @@@
@Test
public void testNoActionTakenOnNullVersion() {
- expect(serviceComponent.isVersionAdvertised()).andReturn(true);
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
resetAll();
replayAll();
@@@ -200,11 -227,10 +212,11 @@@
@Test
public void testSetUpgradeStateToCompleteWhenUpgradeIsInProgressAndNewVersionIsEqualToComponentDesiredVersion() {
- expect(cluster.getUpgradeInProgress()).andReturn(EasyMock.niceMock(UpgradeEntity.class)).atLeastOnce();
- expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce();
- expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS).atLeastOnce();
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
+ expect(cluster.getUpgradeInProgress()).andReturn(DUMMY_UPGRADE_ENTITY);
+
+ expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION);
+ expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.COMPLETE);
expectLastCall().once();
@@@ -215,24 -241,11 +227,24 @@@
}
@Test
+ public void testSetUpgradeStateToNoneWhenNoUpgradeAndNewVersionIsEqualToComponentDesiredVersion() {
+ expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION);
+ expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
+ sch.setUpgradeState(UpgradeState.NONE);
+ expectLastCall().once();
+
+ expect(serviceComponent.getDesiredVersion()).andStubReturn(VALID_NEW_VERSION);
+ replayAll();
+
+ sendEventAndVerify(VALID_NEW_VERSION);
+ }
+
+ @Test
public void testSetUpgradeStateToVersionMismatchWhenUpgradeIsInProgressAndNewVersionIsNotEqualToComponentDesiredVersion() {
- expect(cluster.getUpgradeInProgress()).andReturn(createNiceMock(UpgradeEntity.class)).atLeastOnce();
- expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce();
- expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS).atLeastOnce();
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
+ expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION);
+ expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.VERSION_MISMATCH);
expectLastCall().once();
@@@ -247,11 -257,11 +259,11 @@@
@Test
public void testSetUpgradeStateToCompleteWhenHostHasVersionMismatchAndNewVersionIsEqualToComponentDesiredVersionAndClusterUpgradeIsInProgress() {
- expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce();
- expect(sch.getUpgradeState()).andReturn(UpgradeState.VERSION_MISMATCH).atLeastOnce();
- expect(cluster.getUpgradeInProgress()).andReturn(DUMMY_UPGRADE_ENTITY).atLeastOnce();
+ expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION);
+ expect(sch.getUpgradeState()).andReturn(UpgradeState.VERSION_MISMATCH);
+ expect(cluster.getUpgradeInProgress()).andReturn(DUMMY_UPGRADE_ENTITY);
expect(serviceComponent.getDesiredVersion()).andStubReturn(VALID_NEW_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.COMPLETE);
expectLastCall().once();
@@@ -262,10 -272,10 +274,10 @@@
@Test
public void testSetUpgradeStateToNoneWhenHostHasVersionMismatchAndNewVersionIsEqualToComponentDesiredVersion() {
- expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce();
- expect(sch.getUpgradeState()).andReturn(UpgradeState.VERSION_MISMATCH).atLeastOnce();
+ expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION);
+ expect(sch.getUpgradeState()).andReturn(UpgradeState.VERSION_MISMATCH);
expect(serviceComponent.getDesiredVersion()).andStubReturn(VALID_NEW_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.NONE);
expectLastCall().once();
@@@ -275,9 -285,20 +287,9 @@@
}
@Test
- @Experimental(
- feature = ExperimentalFeature.STACK_UPGRADES_BETWEEN_VENDORS,
- comment = "Version Mismatch happened previously when not in an upgrade")
public void testSetUpgradeStateToVersionMismatchByDefaultWhenHostAndNewVersionsAreValid() {
- expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce();
- expect(serviceComponent.getDesiredVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce();
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
-
- // !!! VERSION_MISMATCH might need to be allowed to happen when not in an
- // upgrade
- expect(cluster.getUpgradeInProgress()).andReturn(
- EasyMock.niceMock(UpgradeEntity.class)).atLeastOnce();
- expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS).atLeastOnce();
-
+ expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
sch.setUpgradeState(UpgradeState.VERSION_MISMATCH);
expectLastCall().once();
@@@ -288,8 -309,8 +300,8 @@@
@Test
public void testSetRepositoryVersion() throws Exception {
- expect(sch.getVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce();
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
+ expect(sch.getVersion()).andReturn(UNKNOWN_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE);
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
RepositoryVersionDAO dao = createNiceMock(RepositoryVersionDAO.class);
RepositoryVersionEntity entity = createNiceMock(RepositoryVersionEntity.class);
@@@ -321,8 -342,12 +333,10 @@@
*/
@Test
public void testRepositoryVersionNotSetDuringUpgrade() throws Exception {
- expect(sch.getVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce();
- expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
++ expect(componentInfo.isVersionAdvertised()).andReturn(true).once();
+
// this call will make it seem like there is an upgrade in progress
- expect(cluster.getUpgradeInProgress()).andReturn(
- createNiceMock(UpgradeEntity.class)).atLeastOnce();
+ expect(cluster.getUpgradeInProgress()).andReturn(createNiceMock(UpgradeEntity.class));
// create the DAO - nothing will be called on it, so make it strict
RepositoryVersionDAO dao = createStrictMock(RepositoryVersionDAO.class);
[19/23] ambari git commit: AMBARI-21534 Spinner doesnt disappear and
hosts not loading even after 5 minutes. (atkach)
Posted by jo...@apache.org.
AMBARI-21534 Spinner doesnt disappear and hosts not loading even after 5 minutes. (atkach)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4078c486
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4078c486
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4078c486
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 4078c48679aaea2f4d8636d141c2c1ca8cb68145
Parents: f9ce03e
Author: Andrii Tkach <at...@apache.org>
Authored: Thu Jul 20 14:50:38 2017 +0300
Committer: Andrii Tkach <at...@apache.org>
Committed: Thu Jul 20 14:50:38 2017 +0300
----------------------------------------------------------------------
ambari-web/app/mappers/hosts_mapper.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4078c486/ambari-web/app/mappers/hosts_mapper.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mappers/hosts_mapper.js b/ambari-web/app/mappers/hosts_mapper.js
index 4d6618f..5042e67 100644
--- a/ambari-web/app/mappers/hosts_mapper.js
+++ b/ambari-web/app/mappers/hosts_mapper.js
@@ -172,7 +172,7 @@ App.hostsMapper = App.QuickDataMapper.create({
for (var j = 0; j < item.stack_versions.length; j++) {
var stackVersion = item.stack_versions[j];
var versionNumber = Em.get(stackVersion.repository_versions[0], 'RepositoryVersions.repository_version');
- var isDifferentStack = stackVersion.HostStackVersions.stack !== currentVersion.HostStackVersions.stack;
+ var isDifferentStack = currentVersion && (stackVersion.HostStackVersions.stack !== currentVersion.HostStackVersions.stack);
var isCompatible = App.RepositoryVersion.find(Em.get(stackVersion.repository_versions[0], 'RepositoryVersions.id')).get('isCompatible');
stackVersion.host_name = item.Hosts.host_name;
if (isDifferentStack && !isCompatible) {
[08/23] ambari git commit: AMBARI-21514 Remove IOP select button not
visible after HDP 2.6.2 is registered. (atkach)
Posted by jo...@apache.org.
AMBARI-21514 Remove IOP select button not visible after HDP 2.6.2 is registered. (atkach)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7542bc39
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7542bc39
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7542bc39
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 7542bc3918204e896d3b64bbac8ed7186939d515
Parents: 837b0bd
Author: Andrii Tkach <at...@apache.org>
Authored: Wed Jul 19 13:59:45 2017 +0300
Committer: Andrii Tkach <at...@apache.org>
Committed: Wed Jul 19 13:59:45 2017 +0300
----------------------------------------------------------------------
.../stack_upgrade/upgrade_version_box_view.js | 31 ++++++++++----
.../upgrade_version_column_view.js | 38 -----------------
.../upgrade_version_box_view_test.js | 43 ++++++++++++++++++++
3 files changed, 66 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7542bc39/ambari-web/app/views/main/admin/stack_upgrade/upgrade_version_box_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/main/admin/stack_upgrade/upgrade_version_box_view.js b/ambari-web/app/views/main/admin/stack_upgrade/upgrade_version_box_view.js
index 956339f..bbd1415 100644
--- a/ambari-web/app/views/main/admin/stack_upgrade/upgrade_version_box_view.js
+++ b/ambari-web/app/views/main/admin/stack_upgrade/upgrade_version_box_view.js
@@ -117,7 +117,7 @@ App.UpgradeVersionBoxView = Em.View.extend({
},
'INIT': {
isButton: true,
- text: Em.I18n.t('admin.stackVersions.version.installNow'),
+ text: Em.I18n.t('common.install'),
action: 'installRepoVersionConfirmation'
},
'LOADING': {
@@ -173,6 +173,10 @@ App.UpgradeVersionBoxView = Em.View.extend({
else if (status === 'INIT') {
requestInProgressRepoId && requestInProgressRepoId == this.get('content.id') ? element.setProperties(statePropertiesMap['LOADING']) : element.setProperties(statePropertiesMap[status]);
element.set('isDisabled', this.isDisabledOnInit());
+ if (this.addRemoveIopSelectButton(element, this.isDisabledOnInit())) {
+ element.set('isButtonGroup', true);
+ element.set('isButton', false);
+ }
}
else if ((status === 'INSTALLED' && !this.get('isUpgrading')) ||
(['INSTALL_FAILED', 'OUT_OF_SYNC'].contains(status))) {
@@ -195,13 +199,7 @@ App.UpgradeVersionBoxView = Em.View.extend({
action: 'installRepoVersionConfirmation',
isDisabled: isDisabled
});
- if (App.get('currentStackName') === 'BigInsights' && !App.get('upgradeIsRunning')) {
- element.get('buttons').pushObject({
- text: Em.I18n.t('admin.stackVersions.removeIopSelect'),
- action: 'removeIopSelect',
- isDisabled: isDisabled
- });
- }
+ this.addRemoveIopSelectButton(element, isDisabled);
}
element.set('isDisabled', isDisabled);
}
@@ -254,6 +252,23 @@ App.UpgradeVersionBoxView = Em.View.extend({
),
/**
+ * @param {Em.Object} element
+ * @param {boolean} isDisabled
+ * @returns {boolean}
+ */
+ addRemoveIopSelectButton: function(element, isDisabled) {
+ if (App.get('currentStackName') === 'BigInsights' && !App.get('upgradeIsRunning')) {
+ element.get('buttons').pushObject({
+ text: Em.I18n.t('admin.stackVersions.removeIopSelect'),
+ action: 'removeIopSelect',
+ isDisabled: isDisabled
+ });
+ return true;
+ }
+ return false;
+ },
+
+ /**
* check if actions of INIT stack version disabled
* @returns {boolean}
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/7542bc39/ambari-web/app/views/main/admin/stack_upgrade/upgrade_version_column_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/main/admin/stack_upgrade/upgrade_version_column_view.js b/ambari-web/app/views/main/admin/stack_upgrade/upgrade_version_column_view.js
index f6f86db..66e424e 100644
--- a/ambari-web/app/views/main/admin/stack_upgrade/upgrade_version_column_view.js
+++ b/ambari-web/app/views/main/admin/stack_upgrade/upgrade_version_column_view.js
@@ -64,44 +64,6 @@ App.UpgradeVersionColumnView = App.UpgradeVersionBoxView.extend({
}.property(),
/**
- * map of properties which correspond to particular state of Upgrade version
- * @type {object}
- */
- statePropertiesMap: {
- 'CURRENT': {
- isLabel: true,
- text: Em.I18n.t('common.current'),
- class: 'label label-success'
- },
- 'INIT': {
- isButton: true,
- text: Em.I18n.t('common.install'),
- action: 'installRepoVersionConfirmation'
- },
- 'LOADING': {
- isSpinner: true,
- class: 'spinner'
- },
- 'INSTALLING': {
- iconClass: 'icon-cog',
- isLink: true,
- text: Em.I18n.t('hosts.host.stackVersions.status.installing'),
- action: 'showProgressPopup'
- },
- 'INSTALLED': {
- iconClass: 'icon-ok',
- isLink: true,
- text: Em.I18n.t('common.installed'),
- action: null
- },
- 'SUSPENDED': {
- isButton: true,
- text: Em.I18n.t('admin.stackUpgrade.dialog.resume'),
- action: 'resumeUpgrade'
- }
- },
-
- /**
* on click handler for "show details" link
*/
openVersionBoxPopup: function (event) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/7542bc39/ambari-web/test/views/main/admin/stack_upgrade/upgrade_version_box_view_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/views/main/admin/stack_upgrade/upgrade_version_box_view_test.js b/ambari-web/test/views/main/admin/stack_upgrade/upgrade_version_box_view_test.js
index 971b1f3..b928b02 100644
--- a/ambari-web/test/views/main/admin/stack_upgrade/upgrade_version_box_view_test.js
+++ b/ambari-web/test/views/main/admin/stack_upgrade/upgrade_version_box_view_test.js
@@ -1137,4 +1137,47 @@ describe('App.UpgradeVersionBoxView', function () {
});
});
});
+
+ describe('#addRemoveIopSelectButton', function() {
+ beforeEach(function() {
+ this.mock = sinon.stub(App, 'get');
+ });
+ afterEach(function() {
+ this.mock.restore();
+ });
+
+ it('should add remove Iop Select button', function() {
+ this.mock.withArgs('currentStackName').returns('BigInsights');
+ this.mock.withArgs('upgradeIsRunning').returns(false);
+ var element = Em.Object.create({
+ buttons: []
+ });
+ view.addRemoveIopSelectButton(element, true);
+ expect(element.get('buttons')[0]).to.be.eql({
+ text: Em.I18n.t('admin.stackVersions.removeIopSelect'),
+ action: 'removeIopSelect',
+ isDisabled: true
+ });
+ });
+
+ it('should not add remove Iop Select button when upgrade is running', function() {
+ this.mock.withArgs('currentStackName').returns('BigInsights-1');
+ this.mock.withArgs('upgradeIsRunning').returns(true);
+ var element = Em.Object.create({
+ buttons: []
+ });
+ view.addRemoveIopSelectButton(element, true);
+ expect(element.get('buttons')).to.be.empty;
+ });
+
+ it('should not add remove Iop Select button when stack not BigInsights', function() {
+ this.mock.withArgs('currentStackName').returns('HDP');
+ this.mock.withArgs('upgradeIsRunning').returns(false);
+ var element = Em.Object.create({
+ buttons: []
+ });
+ view.addRemoveIopSelectButton(element, true);
+ expect(element.get('buttons')).to.be.empty;
+ });
+ });
});
[15/23] ambari git commit: AMBARI-21092. HSI start failed due to
Unrecognized VM option 'UseParallelGC-Xss512k' during EU (missing space).
(sseth via Swapan Shridhar.
Posted by jo...@apache.org.
AMBARI-21092. HSI start failed due to Unrecognized VM option 'UseParallelGC-Xss512k' during EU (missing space). (sseth via Swapan Shridhar.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fb2c2029
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fb2c2029
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fb2c2029
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: fb2c202996d5b0c197c1e7a475ba7eb0be8742a1
Parents: 037d392
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Wed Jul 19 14:57:40 2017 -0700
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Wed Jul 19 14:59:40 2017 -0700
----------------------------------------------------------------------
.../src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/fb2c2029/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 6166318..f227bac 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -524,7 +524,7 @@
<definition xsi:type="configure" id="llap_append_stack_size_java_opts" summary="Update JVM stack size for LLAP">
<type>hive-interactive-env</type>
- <insert key="llap_java_opts" value="-Xss512k" insert-type="append" newline-before="false" newline-after="false" />
+ <insert key="llap_java_opts" value=" -Xss512k " insert-type="append" newline-before="false" newline-after="false" />
</definition>
<definition xsi:type="configure" id="llap_update_shuffle_parallel_copies" summary="Update tez shuffle parallel copies for LLAP">
[22/23] ambari git commit: AMBARI-21527 Restart of MR2 History Server
failed due to wrong NameNode RPC address (dili)
Posted by jo...@apache.org.
AMBARI-21527 Restart of MR2 History Server failed due to wrong NameNode RPC address (dili)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/212ee1cb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/212ee1cb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/212ee1cb
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 212ee1cb04483a2cd7fafb4304a1c5879f2895dc
Parents: 86787c3
Author: Di Li <di...@apache.org>
Authored: Thu Jul 20 14:21:57 2017 -0400
Committer: Di Li <di...@apache.org>
Committed: Thu Jul 20 14:21:57 2017 -0400
----------------------------------------------------------------------
.../stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml | 7 +++++++
.../4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 5 +++++
.../stacks/BigInsights/4.2/upgrades/config-upgrade.xml | 7 +++++++
.../4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 5 +++++
4 files changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/212ee1cb/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
index e476d57..c8549b3 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
@@ -26,6 +26,13 @@
<replace key="content" find="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/iop/current/hadoop-client/lib/native/Linux-amd64-64" replace-with="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/hdp/current/hadoop-client/lib/native" />
<replace key="content" find="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}" replace-with="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} if [ -d "/usr/lib/hadoop-lzo" ]; then export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/hadoop-lzo/lib/* export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop-lzo/lib/native fi"/>
</definition>
+ <definition xsi:type="configure" id="biginsights_4_2_namenode_adjustments">
+ <type>hdfs-site</type>
+ <transfer operation="delete" delete-key="dfs.namenode.rpc-address" if-type="hdfs-site" if-key="dfs.nameservices" if-key-state="present"/>
+ <transfer operation="delete" delete-key="dfs.namenode.http-address" if-type="hdfs-site" if-key="dfs.nameservices" if-key-state="present"/>
+ <transfer operation="delete" delete-key="dfs.namenode.https-address" if-type="hdfs-site" if-key="dfs.nameservices" if-key-state="present"/>
+ <transfer operation="delete" delete-key="dfs.namenode.rpc-address" if-type="core-site" if-key="fs.defaultFS" if-key-state="present"/> <!-- Make sure to use the existing fs.defaultFS for the non-HA case -->
+ </definition>
</changes>
</component>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/212ee1cb/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
index cbd0550..57e731e 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
@@ -200,6 +200,11 @@
<task xsi:type="configure" id="biginsights_4_2_namenode_update_hadoop_env" />
</execute-stage>
+ <!-- HDFS -->
+ <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for Hadoop NameNode">
+ <task xsi:type="configure" id="biginsights_4_2_namenode_adjustments" />
+ </execute-stage>
+
<!-- YARN -->
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Update Yarn configurations">
<task xsi:type="configure" id="biginsights_4_2_yarn_config_update" />
http://git-wip-us.apache.org/repos/asf/ambari/blob/212ee1cb/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
index dada6e2..bf6dd6b 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
@@ -118,6 +118,13 @@
<replace key="content" find="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/iop/current/hadoop-client/lib/native/Linux-amd64-64" replace-with="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/hdp/current/hadoop-client/lib/native" />
<replace key="content" find="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}" replace-with="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} if [ -d "/usr/lib/hadoop-lzo" ]; then export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/hadoop-lzo/lib/* export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop-lzo/lib/native fi"/>
</definition>
+ <definition xsi:type="configure" id="biginsights_4_2_namenode_adjustments">
+ <type>hdfs-site</type>
+ <transfer operation="delete" delete-key="dfs.namenode.rpc-address" if-type="hdfs-site" if-key="dfs.nameservices" if-key-state="present"/>
+ <transfer operation="delete" delete-key="dfs.namenode.http-address" if-type="hdfs-site" if-key="dfs.nameservices" if-key-state="present"/>
+ <transfer operation="delete" delete-key="dfs.namenode.https-address" if-type="hdfs-site" if-key="dfs.nameservices" if-key-state="present"/>
+ <transfer operation="delete" delete-key="dfs.namenode.rpc-address" if-type="core-site" if-key="fs.defaultFS" if-key-state="present"/> <!-- Make sure to use the existing fs.defaultFS for the non-HA case -->
+ </definition>
</changes>
</component>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/212ee1cb/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
index 3ea20ed..b6d39ea 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
@@ -195,6 +195,11 @@
<task xsi:type="configure" id="biginsights_4_2_namenode_update_hadoop_env" />
</execute-stage>
+ <!-- HDFS -->
+ <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for Hadoop NameNode">
+ <task xsi:type="configure" id="biginsights_4_2_namenode_adjustments" />
+ </execute-stage>
+
<!-- YARN -->
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Update Yarn configurations">
<task xsi:type="configure" id="biginsights_4_2_yarn_config_update" />
[13/23] ambari git commit: AMBARI-21524: ResourceManager HA status
not reported when using VIPs (jluniya)
Posted by jo...@apache.org.
AMBARI-21524: ResourceManager HA status not reported when using VIPs (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f169fa9f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f169fa9f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f169fa9f
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: f169fa9f8f253fc46b459eacf8c9d2af1574ab74
Parents: 444115a
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Wed Jul 19 12:23:49 2017 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Wed Jul 19 12:25:03 2017 -0700
----------------------------------------------------------------------
.../internal/AbstractProviderModule.java | 1 +
.../internal/HttpPropertyProvider.java | 27 ++++++++++++++++++--
.../internal/HttpPropertyProviderTest.java | 11 ++++++++
3 files changed, 37 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f169fa9f/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
index 74f9953..5fc4c31 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
@@ -816,6 +816,7 @@ public abstract class AbstractProviderModule implements ProviderModule,
managementController.getClusters(),
PropertyHelper.getPropertyId("HostRoles", "cluster_name"),
PropertyHelper.getPropertyId("HostRoles", "host_name"),
+ PropertyHelper.getPropertyId("HostRoles", "public_host_name"),
PropertyHelper.getPropertyId("HostRoles", "component_name"),
HTTP_PROPERTY_REQUESTS));
http://git-wip-us.apache.org/repos/asf/ambari/blob/f169fa9f/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpPropertyProvider.java
index c53df3f..7380102 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpPropertyProvider.java
@@ -48,6 +48,7 @@ public class HttpPropertyProvider extends BaseProvider implements PropertyProvid
private final StreamProvider streamProvider;
private final String clusterNamePropertyId;
private final String hostNamePropertyId;
+ private final String publicHostNamePropertyId;
private final String componentNamePropertyId;
private final Clusters clusters;
private final Map<String, List<HttpPropertyRequest>> httpPropertyRequests;
@@ -60,6 +61,7 @@ public class HttpPropertyProvider extends BaseProvider implements PropertyProvid
Clusters clusters,
String clusterNamePropertyId,
String hostNamePropertyId,
+ String publicHostNamePropertyId,
String componentNamePropertyId,
Map<String, List<HttpPropertyRequest>> httpPropertyRequests) {
@@ -67,6 +69,7 @@ public class HttpPropertyProvider extends BaseProvider implements PropertyProvid
this.streamProvider = stream;
this.clusterNamePropertyId = clusterNamePropertyId;
this.hostNamePropertyId = hostNamePropertyId;
+ this.publicHostNamePropertyId = publicHostNamePropertyId;
this.componentNamePropertyId = componentNamePropertyId;
this.clusters = clusters;
this.httpPropertyRequests = httpPropertyRequests;
@@ -103,6 +106,7 @@ public class HttpPropertyProvider extends BaseProvider implements PropertyProvid
for (Resource resource : resources) {
String clusterName = (String) resource.getPropertyValue(clusterNamePropertyId);
String hostName = (String) resource.getPropertyValue(hostNamePropertyId);
+ String publicHostName = (String) resource.getPropertyValue(publicHostNamePropertyId);
String componentName = (String) resource.getPropertyValue(componentNamePropertyId);
if (clusterName != null && hostName != null && componentName != null &&
@@ -114,7 +118,7 @@ public class HttpPropertyProvider extends BaseProvider implements PropertyProvid
List<HttpPropertyRequest> httpPropertyRequestList = httpPropertyRequests.get(componentName);
for (HttpPropertyRequest httpPropertyRequest : httpPropertyRequestList) {
- populateResource(httpPropertyRequest, resource, cluster, hostName);
+ populateResource(httpPropertyRequest, resource, cluster, hostName, publicHostName);
}
} catch (AmbariException e) {
String msg = String.format("Could not load cluster with name %s.", clusterName);
@@ -128,7 +132,7 @@ public class HttpPropertyProvider extends BaseProvider implements PropertyProvid
// populate the given resource from the given HTTP property request.
private void populateResource(HttpPropertyRequest httpPropertyRequest, Resource resource,
- Cluster cluster, String hostName) throws SystemException {
+ Cluster cluster, String hostName, String publicHostName) throws SystemException {
String url = httpPropertyRequest.getUrl(cluster, hostName);
@@ -146,6 +150,25 @@ public class HttpPropertyProvider extends BaseProvider implements PropertyProvid
}
} catch (Exception e) {
LOG.debug(String.format("Error reading HTTP response from %s", url), e);
+ if(publicHostName != null && !publicHostName.equalsIgnoreCase(hostName)) {
+ String publicUrl = httpPropertyRequest.getUrl(cluster, publicHostName);
+ LOG.debug(String.format("Retry using public host name url %s", publicUrl));
+ try {
+ InputStream inputStream = streamProvider.readFrom(publicUrl);
+
+ try {
+ httpPropertyRequest.populateResource(resource, inputStream);
+ } finally {
+ try {
+ inputStream.close();
+ } catch (IOException ioe) {
+ LOG.error(String.format("Error closing HTTP response stream %s", url), ioe);
+ }
+ }
+ } catch (Exception ex) {
+ LOG.debug(String.format("Error reading HTTP response from public host name url %s", url), ex);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f169fa9f/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HttpPropertyProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HttpPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HttpPropertyProviderTest.java
index c4fcca0..da054c2 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HttpPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HttpPropertyProviderTest.java
@@ -44,6 +44,7 @@ import org.junit.Test;
public class HttpPropertyProviderTest {
private static final String PROPERTY_ID_CLUSTER_NAME = PropertyHelper.getPropertyId("HostRoles", "cluster_name");
private static final String PROPERTY_ID_HOST_NAME = PropertyHelper.getPropertyId("HostRoles", "host_name");
+ private static final String PROPERTY_ID_PUBLIC_HOST_NAME = PropertyHelper.getPropertyId("HostRoles", "public_host_name");
private static final String PROPERTY_ID_COMPONENT_NAME = PropertyHelper.getPropertyId("HostRoles", "component_name");
private static final String PROPERTY_ID_STALE_CONFIGS = PropertyHelper.getPropertyId(
@@ -85,12 +86,14 @@ public class HttpPropertyProviderTest {
streamProvider, clusters,
PROPERTY_ID_CLUSTER_NAME,
PROPERTY_ID_HOST_NAME,
+ PROPERTY_ID_PUBLIC_HOST_NAME,
PROPERTY_ID_COMPONENT_NAME,
HTTP_PROPERTY_REQUESTS);
Resource resource = new ResourceImpl(Resource.Type.HostComponent);
resource.setProperty(PROPERTY_ID_HOST_NAME, "ec2-54-234-33-50.compute-1.amazonaws.com");
+ resource.setProperty(PROPERTY_ID_PUBLIC_HOST_NAME, "ec2-54-234-33-50.compute-1.amazonaws.com");
resource.setProperty(PROPERTY_ID_CLUSTER_NAME, "testCluster");
resource.setProperty(PROPERTY_ID_COMPONENT_NAME, "RESOURCEMANAGER");
@@ -134,12 +137,14 @@ public class HttpPropertyProviderTest {
streamProvider, clusters,
PROPERTY_ID_CLUSTER_NAME,
PROPERTY_ID_HOST_NAME,
+ PROPERTY_ID_PUBLIC_HOST_NAME,
PROPERTY_ID_COMPONENT_NAME,
HTTP_PROPERTY_REQUESTS);
Resource resource = new ResourceImpl(Resource.Type.HostComponent);
resource.setProperty(PROPERTY_ID_HOST_NAME, "lc6402.ambari.apache.org");
+ resource.setProperty(PROPERTY_ID_PUBLIC_HOST_NAME, "lc6402.ambari.apache.org");
resource.setProperty(PROPERTY_ID_CLUSTER_NAME, "testCluster");
resource.setProperty(PROPERTY_ID_COMPONENT_NAME, "RESOURCEMANAGER");
@@ -174,6 +179,7 @@ public class HttpPropertyProviderTest {
streamProvider, clusters,
PROPERTY_ID_CLUSTER_NAME,
PROPERTY_ID_HOST_NAME,
+ PROPERTY_ID_PUBLIC_HOST_NAME,
PROPERTY_ID_COMPONENT_NAME,
HTTP_PROPERTY_REQUESTS);
@@ -181,6 +187,7 @@ public class HttpPropertyProviderTest {
resource.setProperty(PROPERTY_ID_CLUSTER_NAME, "testCluster");
resource.setProperty(PROPERTY_ID_HOST_NAME, "ec2-54-234-33-50.compute-1.amazonaws.com");
+ resource.setProperty(PROPERTY_ID_PUBLIC_HOST_NAME, "ec2-54-234-33-50.compute-1.amazonaws.com");
resource.setProperty(PROPERTY_ID_COMPONENT_NAME, "ATLAS_SERVER");
Request request = PropertyHelper.getReadRequest(Collections.<String>emptySet());
@@ -214,6 +221,7 @@ public class HttpPropertyProviderTest {
streamProvider, clusters,
PROPERTY_ID_CLUSTER_NAME,
PROPERTY_ID_HOST_NAME,
+ PROPERTY_ID_PUBLIC_HOST_NAME,
PROPERTY_ID_COMPONENT_NAME,
HTTP_PROPERTY_REQUESTS);
@@ -221,6 +229,7 @@ public class HttpPropertyProviderTest {
resource.setProperty(PROPERTY_ID_CLUSTER_NAME, "testCluster");
resource.setProperty(PROPERTY_ID_HOST_NAME, "ec2-54-234-33-50.compute-1.amazonaws.com");
+ resource.setProperty(PROPERTY_ID_PUBLIC_HOST_NAME, "ec2-54-234-33-50.compute-1.amazonaws.com");
resource.setProperty(PROPERTY_ID_COMPONENT_NAME, "ATLAS_SERVER");
Request request = PropertyHelper.getReadRequest(Collections.<String>emptySet());
@@ -249,12 +258,14 @@ public class HttpPropertyProviderTest {
streamProvider, clusters,
PROPERTY_ID_CLUSTER_NAME,
PROPERTY_ID_HOST_NAME,
+ PROPERTY_ID_PUBLIC_HOST_NAME,
PROPERTY_ID_COMPONENT_NAME,
HTTP_PROPERTY_REQUESTS);
Resource resource = new ResourceImpl(Resource.Type.HostComponent);
resource.setProperty(PROPERTY_ID_HOST_NAME, "ec2-54-234-33-50.compute-1.amazonaws.com");
+ resource.setProperty(PROPERTY_ID_PUBLIC_HOST_NAME, "ec2-54-234-33-50.compute-1.amazonaws.com");
resource.setProperty(PROPERTY_ID_CLUSTER_NAME, "testCluster");
resource.setProperty(PROPERTY_ID_COMPONENT_NAME, componentName);
[14/23] ambari git commit: AMBARI-21522 - Installation Commands On
New Clusters Don't Send Down Correct Versions (jonathanhurley)
Posted by jo...@apache.org.
AMBARI-21522 - Installation Commands On New Clusters Don't Send Down Correct Versions (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/037d3926
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/037d3926
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/037d3926
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 037d3926081cf79383466933f4495fdfe9bde6d5
Parents: f169fa9
Author: Dmytro Sen <ds...@apache.org>
Authored: Wed Jul 19 18:38:00 2017 +0300
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Wed Jul 19 16:31:29 2017 -0400
----------------------------------------------------------------------
.../actionmanager/ExecutionCommandWrapper.java | 29 ++++-
.../0.1.0.2.3/package/scripts/atlas_client.py | 2 +-
.../ATLAS/0.1.0.2.3/package/scripts/params.py | 2 +
.../ExecutionCommandWrapperTest.java | 112 ++++++++++++++++---
4 files changed, 120 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/037d3926/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
index 8875314..41521f8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
@@ -20,6 +20,7 @@ package org.apache.ambari.server.actionmanager;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -27,6 +28,7 @@ import java.util.TreeMap;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.ClusterNotFoundException;
+import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.agent.AgentCommand.AgentCommandType;
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.agent.ExecutionCommand.KeyNames;
@@ -199,15 +201,30 @@ public class ExecutionCommandWrapper {
Map<String,String> commandParams = executionCommand.getCommandParams();
// set the version for the command if it's not already set
- ClusterVersionEntity effectiveClusterVersion = cluster.getEffectiveClusterVersion();
- if (null != effectiveClusterVersion && !commandParams.containsKey(KeyNames.VERSION)) {
- commandParams.put(KeyNames.VERSION,
- effectiveClusterVersion.getRepositoryVersion().getVersion());
+ if (!commandParams.containsKey(KeyNames.VERSION)) {
+ // the cluster's effective version should be used for this command
+ ClusterVersionEntity effectiveClusterVersion = cluster.getEffectiveClusterVersion();
+
+ // in the event that the effective version is NULL (meaning that most
+ // likely the cluster is still being provisioned), then send down the
+ // version if this is not an install command
+ if (null == effectiveClusterVersion
+ && executionCommand.getRoleCommand() != RoleCommand.INSTALL) {
+ Collection<ClusterVersionEntity> clusterVersions = cluster.getAllClusterVersions();
+ if (clusterVersions.size() == 1) {
+ effectiveClusterVersion = clusterVersions.iterator().next();
+ }
+ }
+
+ if (null != effectiveClusterVersion) {
+ commandParams.put(KeyNames.VERSION,
+ effectiveClusterVersion.getRepositoryVersion().getVersion());
+ }
}
// add the stack and common-services folders to the command, but only if
- // they don't exist - they may have been put on here with specific values
- // ahead of time
+ // they don't exist - they may have been put on here with specific
+ // values ahead of time
StackId stackId = cluster.getDesiredStackVersion();
StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(),
stackId.getStackVersion());
http://git-wip-us.apache.org/repos/asf/ambari/blob/037d3926/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
index 2376ff9..d01ff84 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
@@ -37,7 +37,7 @@ class AtlasClient(Script):
import params
env.set_params(params)
- if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, params.version):
+ if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, params.version_for_stack_feature_checks):
conf_select.select(params.stack_name, "atlas", params.version)
stack_select.select("atlas-client", params.version)
http://git-wip-us.apache.org/repos/asf/ambari/blob/037d3926/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index bdc9ec4..b83a79f 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -28,6 +28,7 @@ from resource_management import format_stack_version, Script
from resource_management.libraries.functions import format
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.stack_features import get_stack_feature_version
from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions.is_empty import is_empty
from resource_management.libraries.functions.expect import expect
@@ -98,6 +99,7 @@ if security_enabled:
# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade
version = default("/commandParams/version", None)
+version_for_stack_feature_checks = get_stack_feature_version(config)
# stack version
stack_version_unformatted = config['hostLevelParams']['stack_version']
http://git-wip-us.apache.org/repos/asf/ambari/blob/037d3926/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
index c28ba5d..25c3381 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
@@ -34,12 +34,16 @@ import org.apache.ambari.server.Role;
import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.agent.AgentCommand.AgentCommandType;
import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.agent.ExecutionCommand.KeyNames;
import org.apache.ambari.server.orm.GuiceJpaInitializer;
import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
+import org.apache.ambari.server.orm.OrmTestHelper;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ConfigFactory;
import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.RepositoryVersionState;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStartEvent;
import org.apache.ambari.server.utils.StageUtils;
@@ -94,6 +98,7 @@ public class ExecutionCommandWrapperTest {
private static ConfigFactory configFactory;
private static ConfigHelper configHelper;
private static StageFactory stageFactory;
+ private static OrmTestHelper ormTestHelper;
@BeforeClass
public static void setup() throws AmbariException {
@@ -102,6 +107,7 @@ public class ExecutionCommandWrapperTest {
configHelper = injector.getInstance(ConfigHelper.class);
configFactory = injector.getInstance(ConfigFactory.class);
stageFactory = injector.getInstance(StageFactory.class);
+ ormTestHelper = injector.getInstance(OrmTestHelper.class);
clusters = injector.getInstance(Clusters.class);
clusters.addHost(HOST1);
@@ -109,37 +115,41 @@ public class ExecutionCommandWrapperTest {
Cluster cluster1 = clusters.getCluster(CLUSTER1);
- SERVICE_SITE_CLUSTER = new HashMap<String, String>();
+ SERVICE_SITE_CLUSTER = new HashMap<>();
SERVICE_SITE_CLUSTER.put(SERVICE_SITE_NAME1, SERVICE_SITE_VAL1);
SERVICE_SITE_CLUSTER.put(SERVICE_SITE_NAME2, SERVICE_SITE_VAL2);
SERVICE_SITE_CLUSTER.put(SERVICE_SITE_NAME3, SERVICE_SITE_VAL3);
SERVICE_SITE_CLUSTER.put(SERVICE_SITE_NAME4, SERVICE_SITE_VAL4);
- SERVICE_SITE_SERVICE = new HashMap<String, String>();
+ SERVICE_SITE_SERVICE = new HashMap<>();
SERVICE_SITE_SERVICE.put(SERVICE_SITE_NAME1, SERVICE_SITE_VAL1_S);
SERVICE_SITE_SERVICE.put(SERVICE_SITE_NAME5, SERVICE_SITE_VAL5_S);
- SERVICE_SITE_HOST = new HashMap<String, String>();
+ SERVICE_SITE_HOST = new HashMap<>();
SERVICE_SITE_HOST.put(SERVICE_SITE_NAME2, SERVICE_SITE_VAL2_H);
SERVICE_SITE_HOST.put(SERVICE_SITE_NAME6, SERVICE_SITE_VAL6_H);
- GLOBAL_CLUSTER = new HashMap<String, String>();
+ GLOBAL_CLUSTER = new HashMap<>();
GLOBAL_CLUSTER.put(GLOBAL_NAME1, GLOBAL_CLUSTER_VAL1);
GLOBAL_CLUSTER.put(GLOBAL_NAME2, GLOBAL_CLUSTER_VAL2);
- CONFIG_ATTRIBUTES = new HashMap<String, Map<String,String>>();
+ CONFIG_ATTRIBUTES = new HashMap<>();
//Cluster level global config
- configFactory.createNew(cluster1, GLOBAL_CONFIG, CLUSTER_VERSION_TAG, GLOBAL_CLUSTER, CONFIG_ATTRIBUTES);
+ configFactory.createNew(cluster1, GLOBAL_CONFIG, CLUSTER_VERSION_TAG, GLOBAL_CLUSTER,
+ CONFIG_ATTRIBUTES);
//Cluster level service config
- configFactory.createNew(cluster1, SERVICE_SITE_CONFIG, CLUSTER_VERSION_TAG, SERVICE_SITE_CLUSTER, CONFIG_ATTRIBUTES);
+ configFactory.createNew(cluster1, SERVICE_SITE_CONFIG, CLUSTER_VERSION_TAG,
+ SERVICE_SITE_CLUSTER, CONFIG_ATTRIBUTES);
//Service level service config
- configFactory.createNew(cluster1, SERVICE_SITE_CONFIG, SERVICE_VERSION_TAG, SERVICE_SITE_SERVICE, CONFIG_ATTRIBUTES);
+ configFactory.createNew(cluster1, SERVICE_SITE_CONFIG, SERVICE_VERSION_TAG,
+ SERVICE_SITE_SERVICE, CONFIG_ATTRIBUTES);
//Host level service config
- configFactory.createNew(cluster1, SERVICE_SITE_CONFIG, HOST_VERSION_TAG, SERVICE_SITE_HOST, CONFIG_ATTRIBUTES);
+ configFactory.createNew(cluster1, SERVICE_SITE_CONFIG, HOST_VERSION_TAG, SERVICE_SITE_HOST,
+ CONFIG_ATTRIBUTES);
ActionDBAccessor db = injector.getInstance(ActionDBAccessorImpl.class);
@@ -155,7 +165,7 @@ public class ExecutionCommandWrapperTest {
RoleCommand.START,
new ServiceComponentHostStartEvent(Role.NAMENODE.toString(),
hostName, System.currentTimeMillis()), clusterName, "HDFS", false, false);
- List<Stage> stages = new ArrayList<Stage>();
+ List<Stage> stages = new ArrayList<>();
stages.add(s);
Request request = new Request(stages, "clusterHostInfo", clusters);
db.persistActions(request);
@@ -165,13 +175,13 @@ public class ExecutionCommandWrapperTest {
public void testGetExecutionCommand() throws JSONException, AmbariException {
- Map<String, Map<String, String>> confs = new HashMap<String, Map<String, String>>();
- Map<String, String> configurationsGlobal = new HashMap<String, String>();
+ Map<String, Map<String, String>> confs = new HashMap<>();
+ Map<String, String> configurationsGlobal = new HashMap<>();
configurationsGlobal.put(GLOBAL_NAME1, GLOBAL_VAL1);
confs.put(GLOBAL_CONFIG, configurationsGlobal);
- Map<String, Map<String, String>> confTags = new HashMap<String, Map<String, String>>();
- Map<String, String> confTagServiceSite = new HashMap<String, String>();
+ Map<String, Map<String, String>> confTags = new HashMap<>();
+ Map<String, String> confTagServiceSite = new HashMap<>();
confTagServiceSite.put("tag", CLUSTER_VERSION_TAG);
confTagServiceSite.put("service_override_tag", SERVICE_VERSION_TAG);
@@ -223,7 +233,7 @@ public class ExecutionCommandWrapperTest {
//Union of all keys of service site configs
- Set<String> serviceSiteKeys = new HashSet<String>();
+ Set<String> serviceSiteKeys = new HashSet<>();
serviceSiteKeys.addAll(SERVICE_SITE_CLUSTER.keySet());
serviceSiteKeys.addAll(SERVICE_SITE_SERVICE.keySet());
serviceSiteKeys.addAll(SERVICE_SITE_HOST.keySet());
@@ -234,7 +244,7 @@ public class ExecutionCommandWrapperTest {
@Test
public void testGetMergedConfig() {
- Map<String, String> baseConfig = new HashMap<String, String>();
+ Map<String, String> baseConfig = new HashMap<>();
baseConfig.put(SERVICE_SITE_NAME1, SERVICE_SITE_VAL1);
baseConfig.put(SERVICE_SITE_NAME2, SERVICE_SITE_VAL2);
@@ -242,7 +252,7 @@ public class ExecutionCommandWrapperTest {
baseConfig.put(SERVICE_SITE_NAME4, SERVICE_SITE_VAL4);
baseConfig.put(SERVICE_SITE_NAME5, SERVICE_SITE_VAL5);
- Map<String, String> overrideConfig = new HashMap<String, String>();
+ Map<String, String> overrideConfig = new HashMap<>();
overrideConfig.put(SERVICE_SITE_NAME2, SERVICE_SITE_VAL2_H);
overrideConfig.put(SERVICE_SITE_NAME6, SERVICE_SITE_VAL6_H);
@@ -252,7 +262,7 @@ public class ExecutionCommandWrapperTest {
overrideConfig);
- Set<String> configsKeys = new HashSet<String>();
+ Set<String> configsKeys = new HashSet<>();
configsKeys.addAll(baseConfig.keySet());
configsKeys.addAll(overrideConfig.keySet());
@@ -266,6 +276,72 @@ public class ExecutionCommandWrapperTest {
Assert.assertEquals(SERVICE_SITE_VAL6_H, mergedConfig.get(SERVICE_SITE_NAME6));
}
+ /**
+ * Test that the execution command wrapper properly sets the version
+ * information when the cluster is in the INSTALLING state.
+ *
+ * @throws JSONException
+ * @throws AmbariException
+ */
+ @Test
+ public void testExecutionCommandHasVersionInfoWithoutCurrentClusterVersion()
+ throws JSONException, AmbariException {
+ Cluster cluster = clusters.getCluster(CLUSTER1);
+
+ StackId stackId = cluster.getDesiredStackVersion();
+ RepositoryVersionEntity repositoryVersion = ormTestHelper.getOrCreateRepositoryVersion(stackId, "0.1-0000");
+
+ cluster.createClusterVersion(stackId, repositoryVersion.getVersion(), "admin",
+ RepositoryVersionState.INSTALLING);
+
+ // first try with an INSTALL command - this should not populate version info
+ ExecutionCommand executionCommand = new ExecutionCommand();
+ Map<String, String> commandParams = new HashMap<>();
+
+ executionCommand.setClusterName(CLUSTER1);
+ executionCommand.setTaskId(1);
+ executionCommand.setRequestAndStage(1, 1);
+ executionCommand.setHostname(HOST1);
+ executionCommand.setRole("NAMENODE");
+ executionCommand.setRoleParams(Collections.<String, String>emptyMap());
+ executionCommand.setRoleCommand(RoleCommand.INSTALL);
+ executionCommand.setServiceName("HDFS");
+ executionCommand.setCommandType(AgentCommandType.EXECUTION_COMMAND);
+ executionCommand.setCommandParams(commandParams);
+
+ String json = StageUtils.getGson().toJson(executionCommand, ExecutionCommand.class);
+ ExecutionCommandWrapper execCommWrap = new ExecutionCommandWrapper(json);
+ injector.injectMembers(execCommWrap);
+
+ ExecutionCommand processedExecutionCommand = execCommWrap.getExecutionCommand();
+ commandParams = processedExecutionCommand.getCommandParams();
+ Assert.assertFalse(commandParams.containsKey(KeyNames.VERSION));
+
+ // now try with a START command which should populate the version even
+ // though the state is INSTALLING
+ executionCommand = new ExecutionCommand();
+ commandParams = new HashMap<>();
+
+ executionCommand.setClusterName(CLUSTER1);
+ executionCommand.setTaskId(1);
+ executionCommand.setRequestAndStage(1, 1);
+ executionCommand.setHostname(HOST1);
+ executionCommand.setRole("NAMENODE");
+ executionCommand.setRoleParams(Collections.<String, String> emptyMap());
+ executionCommand.setRoleCommand(RoleCommand.START);
+ executionCommand.setServiceName("HDFS");
+ executionCommand.setCommandType(AgentCommandType.EXECUTION_COMMAND);
+ executionCommand.setCommandParams(commandParams);
+
+ json = StageUtils.getGson().toJson(executionCommand, ExecutionCommand.class);
+ execCommWrap = new ExecutionCommandWrapper(json);
+ injector.injectMembers(execCommWrap);
+
+ processedExecutionCommand = execCommWrap.getExecutionCommand();
+ commandParams = processedExecutionCommand.getCommandParams();
+ Assert.assertEquals("0.1-0000", commandParams.get(KeyNames.VERSION));
+ }
+
@AfterClass
public static void tearDown() throws AmbariException, SQLException {
H2DatabaseCleaner.clearDatabaseAndStopPersistenceService(injector);
[16/23] ambari git commit: AMBARI-21526 - ZKFC Doesn't Update its
Version On Some Upgrades (jonathanhurley)
Posted by jo...@apache.org.
AMBARI-21526 - ZKFC Doesn't Update its Version On Some Upgrades (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/963f2b4c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/963f2b4c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/963f2b4c
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 963f2b4cf2793abf40625de5da0ab489c28ea72e
Parents: fb2c202
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Wed Jul 19 16:49:01 2017 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Wed Jul 19 20:05:01 2017 -0400
----------------------------------------------------------------------
.../libraries/functions/stack_features.py | 2 +-
.../listeners/upgrade/StackVersionListener.java | 31 ++++++++++--
.../2.1.0.2.0/package/scripts/zkfc_slave.py | 8 +---
.../upgrade/StackVersionListenerTest.java | 50 +++++++++++++-------
4 files changed, 62 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/963f2b4c/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
index 2c66728..3fcce82 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
@@ -122,7 +122,7 @@ def get_stack_feature_version(config):
# if this is not an upgrade, then we take the simple path
if upgrade_direction is None:
Logger.info(
- "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}-> {4}".format(
+ "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3} -> {4}".format(
stack_version, current_cluster_version, command_stack, command_version, version_for_stack_feature_checks))
return version_for_stack_feature_checks
http://git-wip-us.apache.org/repos/asf/ambari/blob/963f2b4c/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
index 3af5520..4183b51 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
@@ -22,6 +22,7 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.EagerSingleton;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent;
import org.apache.ambari.server.events.publishers.VersionEventPublisher;
import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
@@ -29,8 +30,10 @@ import org.apache.ambari.server.orm.entities.ClusterVersionEntity;
import org.apache.ambari.server.orm.entities.HostVersionEntity;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.State;
import org.apache.ambari.server.state.UpgradeState;
import org.apache.commons.lang.StringUtils;
@@ -39,6 +42,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.eventbus.Subscribe;
import com.google.inject.Inject;
+import com.google.inject.Provider;
import com.google.inject.Singleton;
/**
@@ -66,6 +70,13 @@ public class StackVersionListener {
private RepositoryVersionDAO repositoryVersionDAO;
/**
+ * Used for looking up a component's advertising version status given a stack
+ * and name.
+ */
+ @Inject
+ private Provider<AmbariMetaInfo> ambariMetaInfoProvider;
+
+ /**
* Constructor.
*
* @param eventPublisher the publisher
@@ -108,20 +119,30 @@ public class StackVersionListener {
// Update host component version value if needed
try {
- ServiceComponent sc = cluster.getService(sch.getServiceName()).getServiceComponent(
- sch.getServiceComponentName());
+ // get the component information for the desired stack; if a component
+ // moves from UNKNOWN to providing a version, we must do the version
+ // advertised check against the target stack
+ StackId desiredStackId = sch.getDesiredStackVersion();
+
+ AmbariMetaInfo ambariMetaInfo = ambariMetaInfoProvider.get();
+ ComponentInfo componentInfo = ambariMetaInfo.getComponent(desiredStackId.getStackName(),
+ desiredStackId.getStackVersion(), sch.getServiceName(), sch.getServiceComponentName());
// not advertising a version, do nothing
- if (!sc.isVersionAdvertised()) {
+ if (!componentInfo.isVersionAdvertised()) {
// that's odd; a version came back - log it and still do nothing
if (!StringUtils.equalsIgnoreCase(UNKNOWN_VERSION, newVersion)) {
- LOG.debug(
+ LOG.warn(
"ServiceComponent {} doesn't advertise version, however ServiceHostComponent {} on host {} advertised version as {}. Skipping version update",
- sc.getName(), sch.getServiceComponentName(), sch.getHostName(), newVersion);
+ sch.getServiceComponentName(), sch.getServiceComponentName(), sch.getHostName(),
+ newVersion);
}
return;
}
+ ServiceComponent sc = cluster.getService(sch.getServiceName()).getServiceComponent(
+ sch.getServiceComponentName());
+
boolean desiredVersionIsCurrentlyUnknown = StringUtils.equalsIgnoreCase(UNKNOWN_VERSION,
sc.getDesiredVersion());
http://git-wip-us.apache.org/repos/asf/ambari/blob/963f2b4c/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
index dff1548..db68544 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
@@ -30,11 +30,6 @@ from resource_management.core import shell
from resource_management.libraries.functions import conf_select, stack_select
from resource_management.libraries.functions.constants import StackFeature
from resource_management.libraries.functions.check_process_status import check_process_status
-from resource_management.libraries.functions.security_commons import build_expectations
-from resource_management.libraries.functions.security_commons import cached_kinit_executor
-from resource_management.libraries.functions.security_commons import get_params_from_filesystem
-from resource_management.libraries.functions.security_commons import validate_security_config_properties
-from resource_management.libraries.functions.security_commons import FILE_TYPE_XML
from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.script import Script
from resource_management.core.resources.zkmigrator import ZkMigrator
@@ -140,8 +135,7 @@ class ZkfcSlaveDefault(ZkfcSlave):
Logger.info("Executing Stack Upgrade pre-restart")
import params
env.set_params(params)
- if params.version and check_stack_feature(StackFeature.ZKFC_VERSION_ADVERTISED, params.version) \
- and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
+ if check_stack_feature(StackFeature.ZKFC_VERSION_ADVERTISED, params.version_for_stack_feature_checks):
conf_select.select(params.stack_name, "hadoop", params.version)
stack_select.select("hadoop-hdfs-zkfc", params.version)
http://git-wip-us.apache.org/repos/asf/ambari/blob/963f2b4c/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
index f20c46c..3a4f5b9 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
@@ -32,6 +32,7 @@ import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.orm.entities.UpgradeEntity;
import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.state.ServiceComponentHost;
@@ -81,6 +82,12 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Mock
private Provider<AmbariMetaInfo> ambariMetaInfoProvider;
+ @Mock
+ private ComponentInfo componentInfo;
+
+ @Mock
+ private AmbariMetaInfo ambariMetaInfo;
+
@Before
public void setup() throws Exception {
cluster = createNiceMock(Cluster.class);
@@ -89,22 +96,30 @@ public class StackVersionListenerTest extends EasyMockSupport {
serviceComponent = createNiceMock(ServiceComponent.class);
stackId = createNiceMock(StackId.class);
+ expect(cluster.getService(SERVICE_NAME)).andReturn(service).anyTimes();
expect(cluster.getDesiredStackVersion()).andReturn(stackId).anyTimes();
expect(stackId.getStackName()).andReturn(STACK_NAME).anyTimes();
expect(stackId.getStackVersion()).andReturn(STACK_VERSION).anyTimes();
expect(cluster.getClusterId()).andReturn(CLUSTER_ID).atLeastOnce();
- expect(cluster.getService(SERVICE_NAME)).andReturn(service).atLeastOnce();
- expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(serviceComponent).atLeastOnce();
+ expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(serviceComponent).anyTimes();
+ expect(serviceComponent.getDesiredStackVersion()).andReturn(stackId).anyTimes();
+ expect(sch.getDesiredStackVersion()).andReturn(stackId).atLeastOnce();
expect(sch.getServiceName()).andReturn(SERVICE_NAME).atLeastOnce();
expect(sch.getServiceComponentName()).andReturn(SERVICE_COMPONENT_NAME).atLeastOnce();
+
+ expect(ambariMetaInfoProvider.get()).andReturn(ambariMetaInfo).atLeastOnce();
+ expect(ambariMetaInfo.getComponent(STACK_NAME, STACK_VERSION, SERVICE_NAME,
+ SERVICE_COMPONENT_NAME)).andReturn(componentInfo).atLeastOnce();
+
+ injectMocks(listener);
}
@Test
public void testRecalculateHostVersionStateWhenVersionIsNullAndNewVersionIsNotBlank() throws AmbariException {
expect(sch.getVersion()).andReturn(null).atLeastOnce();
expect(serviceComponent.getDesiredVersion()).andReturn(INVALID_NEW_VERSION).atLeastOnce();
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
sch.setVersion(INVALID_NEW_VERSION);
expectLastCall().once();
expect(sch.recalculateHostVersionState()).andReturn(null).once();
@@ -117,7 +132,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testRecalculateHostVersionStateWhenVersionIsUnknownAndNewVersionIsNotBlank() throws AmbariException {
expect(sch.getVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce();
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
sch.setVersion(INVALID_NEW_VERSION);
expectLastCall().once();
expect(sch.recalculateHostVersionState()).andReturn(null).once();
@@ -130,7 +145,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testRecalculateClusterVersionStateWhenVersionIsNullAndNewVersionIsValid() throws AmbariException {
expect(sch.getVersion()).andReturn(null).atLeastOnce();
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
sch.setVersion(VALID_NEW_VERSION);
expectLastCall().once();
expect(sch.recalculateHostVersionState()).andReturn(DUMMY_REPOSITORY_VERSION_ENTITY).once();
@@ -145,7 +160,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testRecalculateClusterVersionStateWhenVersionIsUnknownAndNewVersionIsValid() throws AmbariException {
expect(sch.getVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce();
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
sch.setVersion(VALID_NEW_VERSION);
expectLastCall().once();
expect(sch.recalculateHostVersionState()).andReturn(DUMMY_REPOSITORY_VERSION_ENTITY).once();
@@ -160,7 +175,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testRecalculateHostVersionStateWhenComponentDesiredVersionIsUnknownAndNewVersionIsNotValid() throws AmbariException {
expect(serviceComponent.getDesiredVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce();
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
serviceComponent.setDesiredVersion(INVALID_NEW_VERSION);
expectLastCall().once();
sch.setUpgradeState(UpgradeState.NONE);
@@ -177,7 +192,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testRecalculateClusterVersionStateWhenComponentDesiredVersionIsUnknownAndNewVersionIsValid() throws AmbariException {
expect(serviceComponent.getDesiredVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce();
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
serviceComponent.setDesiredVersion(VALID_NEW_VERSION);
expectLastCall().once();
sch.setUpgradeState(UpgradeState.NONE);
@@ -195,7 +210,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testRecalculateClusterVersionStateWhenVersionNotAdvertised() throws AmbariException {
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.FALSE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(false).atLeastOnce();
replayAll();
sendEventAndVerify(VALID_NEW_VERSION);
}
@@ -203,7 +218,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testNoActionTakenOnNullVersion() {
- expect(serviceComponent.isVersionAdvertised()).andReturn(true);
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
resetAll();
replayAll();
@@ -215,7 +230,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
expect(cluster.getUpgradeInProgress()).andReturn(EasyMock.niceMock(UpgradeEntity.class)).atLeastOnce();
expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce();
expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS).atLeastOnce();
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
sch.setUpgradeState(UpgradeState.COMPLETE);
expectLastCall().once();
@@ -230,7 +245,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
expect(cluster.getUpgradeInProgress()).andReturn(createNiceMock(UpgradeEntity.class)).atLeastOnce();
expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce();
expect(sch.getUpgradeState()).andReturn(UpgradeState.IN_PROGRESS).atLeastOnce();
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
sch.setUpgradeState(UpgradeState.VERSION_MISMATCH);
expectLastCall().once();
@@ -246,7 +261,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
expect(sch.getUpgradeState()).andReturn(UpgradeState.VERSION_MISMATCH).atLeastOnce();
expect(cluster.getUpgradeInProgress()).andReturn(DUMMY_UPGRADE_ENTITY).atLeastOnce();
expect(serviceComponent.getDesiredVersion()).andStubReturn(VALID_NEW_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
sch.setUpgradeState(UpgradeState.COMPLETE);
expectLastCall().once();
@@ -260,7 +275,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce();
expect(sch.getUpgradeState()).andReturn(UpgradeState.VERSION_MISMATCH).atLeastOnce();
expect(serviceComponent.getDesiredVersion()).andStubReturn(VALID_NEW_VERSION);
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
sch.setUpgradeState(UpgradeState.NONE);
expectLastCall().once();
@@ -276,7 +291,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
public void testSetUpgradeStateToVersionMismatchByDefaultWhenHostAndNewVersionsAreValid() {
expect(sch.getVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce();
expect(serviceComponent.getDesiredVersion()).andReturn(VALID_PREVIOUS_VERSION).atLeastOnce();
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
// !!! VERSION_MISMATCH might need to be allowed to happen when not in an
// upgrade
@@ -295,7 +310,7 @@ public class StackVersionListenerTest extends EasyMockSupport {
@Test
public void testSetRepositoryVersion() throws Exception {
expect(sch.getVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce();
- expect(serviceComponent.isVersionAdvertised()).andReturn(Boolean.TRUE).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
RepositoryVersionDAO dao = createNiceMock(RepositoryVersionDAO.class);
RepositoryVersionEntity entity = createNiceMock(RepositoryVersionEntity.class);
@@ -327,6 +342,9 @@ public class StackVersionListenerTest extends EasyMockSupport {
*/
@Test
public void testRepositoryVersionNotSetDuringUpgrade() throws Exception {
+ expect(sch.getVersion()).andReturn(UNKNOWN_VERSION).atLeastOnce();
+ expect(componentInfo.isVersionAdvertised()).andReturn(true).atLeastOnce();
+
// this call will make it seem like there is an upgrade in progress
expect(cluster.getUpgradeInProgress()).andReturn(
createNiceMock(UpgradeEntity.class)).atLeastOnce();
[20/23] ambari git commit: AMBARI-21530 - Service Checks During
Upgrades Should Use Desired Stack (jonathanhurley)
Posted by jo...@apache.org.
AMBARI-21530 - Service Checks During Upgrades Should Use Desired Stack (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/282c4e21
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/282c4e21
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/282c4e21
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 282c4e213056d351331adac498d4655b8ebc9251
Parents: 4078c48
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Wed Jul 19 22:04:07 2017 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Thu Jul 20 08:09:38 2017 -0400
----------------------------------------------------------------------
.../AmbariCustomCommandExecutionHelper.java | 5 --
.../internal/UpgradeResourceProvider.java | 50 +++++++++++-----
.../2.1.0.2.0/package/scripts/historyserver.py | 2 +-
.../2.1.0.2.0/package/scripts/params_linux.py | 11 ++--
.../2.1.0.2.0/package/scripts/service_check.py | 6 +-
.../src/test/python/TestStackFeature.py | 61 ++++++++++++++++++++
6 files changed, 103 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index 2eca76c..723a10d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -27,7 +27,6 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_CO
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION;
@@ -40,7 +39,6 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JD
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_GROUPS;
@@ -772,9 +770,6 @@ public class AmbariCustomCommandExecutionHelper {
}
commandParams.put(COMMAND_TIMEOUT, commandTimeout);
- commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
- commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
-
execCmd.setCommandParams(commandParams);
if (actionParameters != null) { // If defined
http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 0b47ff1..de3225f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -1337,6 +1337,37 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
}
/**
+ * Adds the hooks and service folders based on the effective stack ID and the
+ * name of the service from the wrapper.
+ *
+ * @param wrapper
+ * the stage wrapper to use when detemrining the service name.
+ * @param effectiveStackId
+ * the stack ID to use when getting the hooks and service folders.
+ * @param commandParams
+ * the params to update with the new values
+ * @throws AmbariException
+ */
+ private void applyRepositoryAssociatedParameters(StageWrapper wrapper, StackId effectiveStackId,
+ Map<String, String> commandParams) throws AmbariException {
+ if (CollectionUtils.isNotEmpty(wrapper.getTasks())
+ && wrapper.getTasks().get(0).getService() != null) {
+
+ AmbariMetaInfo ambariMetaInfo = s_metaProvider.get();
+
+ StackInfo stackInfo = ambariMetaInfo.getStack(effectiveStackId.getStackName(),
+ effectiveStackId.getStackVersion());
+
+ String serviceName = wrapper.getTasks().get(0).getService();
+ ServiceInfo serviceInfo = ambariMetaInfo.getService(effectiveStackId.getStackName(),
+ effectiveStackId.getStackVersion(), serviceName);
+
+ commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
+ commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
+ }
+ }
+
+ /**
* Creates an action stage using the {@link #EXECUTE_TASK_ROLE} custom action
* to execute some Python command.
*
@@ -1389,21 +1420,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
// the ru_execute_tasks invokes scripts - it needs information about where
// the scripts live and for that it should always use the target repository
// stack
- if (CollectionUtils.isNotEmpty(wrapper.getTasks())
- && wrapper.getTasks().get(0).getService() != null) {
-
- AmbariMetaInfo ambariMetaInfo = s_metaProvider.get();
-
- StackInfo stackInfo = ambariMetaInfo.getStack(effectiveStackId.getStackName(),
- effectiveStackId.getStackVersion());
-
- String serviceName = wrapper.getTasks().get(0).getService();
- ServiceInfo serviceInfo = ambariMetaInfo.getService(effectiveStackId.getStackName(),
- effectiveStackId.getStackVersion(), serviceName);
-
- params.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
- params.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
- }
+ applyRepositoryAssociatedParameters(wrapper, effectiveStackId, params);
ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
EXECUTE_TASK_ROLE, Collections.singletonList(filter), params);
@@ -1567,6 +1584,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
// Apply additional parameters to the command that come from the stage.
applyAdditionalParameters(wrapper, commandParams);
+ // add things like hooks and service folders based on effective repo
+ applyRepositoryAssociatedParameters(wrapper, effectiveStackId, commandParams);
+
ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
"SERVICE_CHECK", filters, commandParams);
http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
index 7405a4e..36417a8 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
@@ -92,7 +92,7 @@ class HistoryServerDefault(HistoryServer):
env.set_params(params)
self.configure(env) # FOR SECURITY
- if params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major):
+ if check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.version_for_stack_feature_checks):
# MC Hammer said, "Can't touch this"
resource_created = copy_to_hdfs(
"mapreduce",
http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 5f400a3..a4f7ee9 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -66,21 +66,20 @@ tarball_map = default("/configurations/cluster-env/tarball_map", None)
config_path = os.path.join(stack_root, "current/hadoop-client/conf")
config_dir = os.path.realpath(config_path)
+# get the correct version to use for checking stack features
+version_for_stack_feature_checks = get_stack_feature_version(config)
+
# This is expected to be of the form #.#.#.#
stack_version_unformatted = config['hostLevelParams']['stack_version']
-stack_version_formatted_major = format_stack_version(stack_version_unformatted)
stack_version_formatted = functions.get_stack_version('hadoop-yarn-resourcemanager')
-stack_supports_ru = stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted_major)
-stack_supports_timeline_state_store = stack_version_formatted_major and check_stack_feature(StackFeature.TIMELINE_STATE_STORE, stack_version_formatted_major)
+stack_supports_ru = check_stack_feature(StackFeature.ROLLING_UPGRADE, version_for_stack_feature_checks)
+stack_supports_timeline_state_store = check_stack_feature(StackFeature.TIMELINE_STATE_STORE, version_for_stack_feature_checks)
# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade.
# It cannot be used during the initial Cluser Install because the version is not yet known.
version = default("/commandParams/version", None)
-# get the correct version to use for checking stack features
-version_for_stack_feature_checks = get_stack_feature_version(config)
-
stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
index a36e8cc..1193b60 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
@@ -95,11 +95,7 @@ class ServiceCheckDefault(ServiceCheck):
mode=params.smoke_hdfs_user_mode,
)
- if params.stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted_major):
- path_to_distributed_shell_jar = format("{stack_root}/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar")
- else:
- path_to_distributed_shell_jar = "/usr/lib/hadoop-yarn/hadoop-yarn-applications-distributedshell*.jar"
-
+ path_to_distributed_shell_jar = format("{stack_root}/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar")
yarn_distrubuted_shell_check_params = ["yarn org.apache.hadoop.yarn.applications.distributedshell.Client",
"-shell_command", "ls", "-num_containers", "{number_of_nm}",
"-jar", "{path_to_distributed_shell_jar}", "-timeout", "300000",
http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/test/python/TestStackFeature.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestStackFeature.py b/ambari-server/src/test/python/TestStackFeature.py
index 230734c..6e8bcec 100644
--- a/ambari-server/src/test/python/TestStackFeature.py
+++ b/ambari-server/src/test/python/TestStackFeature.py
@@ -21,10 +21,13 @@ limitations under the License.
from resource_management.core.logger import Logger
from resource_management.libraries.functions.stack_features import get_stack_feature_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.script import Script
from resource_management.core.exceptions import Fail
from unittest import TestCase
+import json
+
Logger.initialize_logger()
class TestStackFeature(TestCase):
@@ -115,6 +118,34 @@ class TestStackFeature(TestCase):
stack_feature_version = get_stack_feature_version(command_json)
self.assertEqual("2.5.9.9-9999", stack_feature_version)
+
+ def test_get_stack_feature(self):
+ """
+ Tests the stack feature version calculated during a STOP command in a downgrade.
+ :return:
+ """
+ command_json = TestStackFeature._get_cluster_upgrade_restart_json()
+ Script.config = command_json
+
+ Script.config["configurations"] = {}
+ Script.config["configurations"]["cluster-env"] = {}
+ Script.config["configurations"]["cluster-env"]["stack_features"] = {}
+ Script.config["configurations"]["cluster-env"]["stack_features"] = json.dumps(TestStackFeature._get_stack_feature_json())
+
+ stack_feature_version = get_stack_feature_version(command_json)
+ self.assertTrue(check_stack_feature("stack-feature-1", stack_feature_version))
+ self.assertTrue(check_stack_feature("stack-feature-2", stack_feature_version))
+ self.assertFalse(check_stack_feature("stack-feature-3", stack_feature_version))
+
+ command_json = TestStackFeature._get_cluster_install_command_json()
+ Script.config.update(command_json)
+
+ stack_feature_version = get_stack_feature_version(command_json)
+ self.assertTrue(check_stack_feature("stack-feature-1", stack_feature_version))
+ self.assertTrue(check_stack_feature("stack-feature-2", stack_feature_version))
+ self.assertFalse(check_stack_feature("stack-feature-3", stack_feature_version))
+
+
@staticmethod
def _get_cluster_install_command_json():
"""
@@ -221,4 +252,34 @@ class TestStackFeature(TestCase):
"version":"2.5.9.9-9999",
"downgrade_from_version":"2.5.9.9-9999"
}
+ }
+
+ @staticmethod
+ def _get_stack_feature_json():
+ """
+ A STOP command during a downgrade.
+ :return:
+ """
+ return {
+ "HDP": {
+ "stack_features":[
+ {
+ "name":"stack-feature-1",
+ "description":"Stack Feature 1",
+ "min_version":"2.2.0.0"
+ },
+ {
+ "name":"stack-feature-2",
+ "description":"Stack Feature 2",
+ "min_version":"2.2.0.0",
+ "max_version":"2.6.0.0"
+ },
+ {
+ "name":"stack-feature-3",
+ "description":"Stack Feature 3",
+ "min_version":"2.2.0.0",
+ "max_version":"2.3.0.0"
+ }
+ ]
+ }
}
\ No newline at end of file
[21/23] ambari git commit: AMBARI-21535. ACTIVITY_ANALYZER Install
failed: Error: Unable to run the custom hook script (aonishuk)
Posted by jo...@apache.org.
AMBARI-21535. ACTIVITY_ANALYZER Install failed: Error: Unable to run the custom hook script (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/86787c37
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/86787c37
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/86787c37
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 86787c37f669440a85a0038fe834f267ad992b07
Parents: 282c4e2
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Jul 20 16:21:27 2017 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Jul 20 16:21:27 2017 +0300
----------------------------------------------------------------------
.../hooks/before-ANY/files/changeToSecureUid.sh | 2 ++
.../before-ANY/scripts/shared_initialization.py | 19 ++++++-------------
2 files changed, 8 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/86787c37/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/files/changeToSecureUid.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/files/changeToSecureUid.sh b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/files/changeToSecureUid.sh
index 4663f10..a6b8b77 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/files/changeToSecureUid.sh
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/files/changeToSecureUid.sh
@@ -44,6 +44,8 @@ if [ -z $2 ]; then
fi
echo $newUid
exit 0
+else
+ find_available_uid
fi
if [ $newUid -eq 0 ]
http://git-wip-us.apache.org/repos/asf/ambari/blob/86787c37/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
index aa33261..bcd4b17 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
@@ -44,19 +44,12 @@ def setup_users():
)
for user in params.user_list:
- if params.override_uid == "true":
- User(user,
- uid = get_uid(user),
- gid = params.user_to_gid_dict[user],
- groups = params.user_to_groups_dict[user],
- fetch_nonlocal_groups = params.fetch_nonlocal_groups,
- )
- else:
- User(user,
- gid = params.user_to_gid_dict[user],
- groups = params.user_to_groups_dict[user],
- fetch_nonlocal_groups = params.fetch_nonlocal_groups,
- )
+ User(user,
+ uid = get_uid(user) if params.override_uid == "true" else None,
+ gid = params.user_to_gid_dict[user],
+ groups = params.user_to_groups_dict[user],
+ fetch_nonlocal_groups = params.fetch_nonlocal_groups,
+ )
if params.override_uid == "true":
set_uid(params.smoke_user, params.smoke_user_dirs)
[11/23] ambari git commit: AMBARI-21521 Include/exclude files update
is triggered when master is not started if cluster is deployed via BP (dsen)
Posted by jo...@apache.org.
AMBARI-21521 Include/exclude files update is triggered when master is not started if cluster is deployed via BP (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/05e7043c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/05e7043c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/05e7043c
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 05e7043cb20c2b88ac0442ae28eca1d3dfef7911
Parents: dc9bf08
Author: Dmytro Sen <ds...@apache.org>
Authored: Wed Jul 19 18:38:00 2017 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Wed Jul 19 18:41:59 2017 +0300
----------------------------------------------------------------------
.../controller/AmbariManagementControllerImpl.java | 15 +++++++++++++--
.../AmbariManagementControllerImplTest.java | 1 -
2 files changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/05e7043c/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index cccb4b2..850838d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -3362,9 +3362,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
}
try {
- Service s = cluster.getService(serviceName);
//Filter services whose masters are not started
- if (s.getServiceComponent(masterComponentName).getDesiredState() == State.STARTED) {
+ if (isServiceComponentStartedOnAnyHost(cluster, serviceName, masterComponentName)) {
serviceMasterForDecommissionMap.put(serviceName, masterComponentName);
} else {
LOG.info(String.format("Not adding %s service from include/exclude files refresh map because it's master is not started", serviceName));
@@ -3393,6 +3392,18 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
}
+ private boolean isServiceComponentStartedOnAnyHost(Cluster cluster, String serviceName, String masterComponentName) throws AmbariException {
+ Service service = cluster.getService(serviceName);
+ ServiceComponent serviceComponent = service.getServiceComponent(masterComponentName);
+ Map<String, ServiceComponentHost> schMap = serviceComponent.getServiceComponentHosts();
+ for (ServiceComponentHost sch : schMap.values()) {
+ if (sch.getState() == State.STARTED) {
+ return true;
+ }
+ }
+ return false;
+ }
+
@Override
public RequestStatusResponse createAndPersistStages(Cluster cluster, Map<String, String> requestProperties,
Map<String, String> requestParameters,
http://git-wip-us.apache.org/repos/asf/ambari/blob/05e7043c/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
index 085b746..35aa37f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
@@ -2409,5 +2409,4 @@ public class AmbariManagementControllerImplTest {
verify(injector, clusters, ambariMetaInfo, stackInfo, cluster, repoVersionDAO, repoVersion, clusterVersionDAO);
}
-
}
[12/23] ambari git commit: AMBARI-21487. ADDENDUM - Add SOLR to
BigInsights 4.2.5 stack (oleewere)
Posted by jo...@apache.org.
AMBARI-21487. ADDENDUM - Add SOLR to BigInsights 4.2.5 stack (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/444115a8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/444115a8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/444115a8
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 444115a8fbab5211671a2976d880a6cf0fa469af
Parents: 05e7043
Author: oleewere <ol...@gmail.com>
Authored: Wed Jul 19 20:15:25 2017 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Wed Jul 19 20:15:53 2017 +0200
----------------------------------------------------------------------
.../BigInsights/4.2.5/services/SOLR/configuration/solr-env.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/444115a8/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-env.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-env.xml
index 4d47cf1..45b7e6a 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-env.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/configuration/solr-env.xml
@@ -82,6 +82,7 @@
<name>solr_hdfs_home_dir</name>
<value>/apps/solr/data</value>
<description>A root location in HDFS for Solr to write collection data to. Rather than specifying an HDFS location for the data directory or update log directory, use this to specify one root location and have everything automatically created within this HDFS</description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>solr_ssl_enabled</name>
[10/23] ambari git commit: Reverting the accidental commit of a rust
file (smohanty)
Posted by jo...@apache.org.
Reverting the accidental commit of a rust file (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dc9bf082
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dc9bf082
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dc9bf082
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: dc9bf0821690dd7f677060dca5f3c4bf81c577aa
Parents: 2b24cde
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Jul 19 07:15:05 2017 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Jul 19 07:15:48 2017 -0700
----------------------------------------------------------------------
ambari-server/src/main/java/org/apache/ambari/server/agent/hello.rs | 0
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc9bf082/ambari-server/src/main/java/org/apache/ambari/server/agent/hello.rs
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/hello.rs b/ambari-server/src/main/java/org/apache/ambari/server/agent/hello.rs
deleted file mode 100644
index e69de29..0000000
[17/23] ambari git commit: AMBARI-21531. Client component restart
fails after Ambari upgrade while running custom hook script on Suse 11
(aonishuk)
Posted by jo...@apache.org.
AMBARI-21531. Client component restart fails after Ambari upgrade while running custom hook script on Suse 11 (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9ff3d66a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9ff3d66a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9ff3d66a
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 9ff3d66a683e45b2257584bb65425fae255e5087
Parents: 963f2b4
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Jul 20 13:27:45 2017 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Jul 20 13:27:45 2017 +0300
----------------------------------------------------------------------
.../resource_management/TestUserResource.py | 2 +-
.../python/resource_management/core/base.py | 11 +++++
.../core/providers/accounts.py | 6 ++-
.../core/resources/accounts.py | 4 +-
.../before-ANY/scripts/shared_initialization.py | 12 +++---
.../2.0.6/hooks/before-ANY/test_before_any.py | 45 +++++++++++++-------
6 files changed, 55 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ff3d66a/ambari-agent/src/test/python/resource_management/TestUserResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestUserResource.py b/ambari-agent/src/test/python/resource_management/TestUserResource.py
index 97d992e..8f1df83 100644
--- a/ambari-agent/src/test/python/resource_management/TestUserResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestUserResource.py
@@ -163,7 +163,7 @@ class TestUserResource(TestCase):
getpwnam_mock.return_value = _get_user_entity()
with Environment('/') as env:
- user = User("mapred", action = "create", uid = "1", shell = "/bin/bash")
+ user = User("mapred", action = "create", uid = 1, shell = "/bin/bash")
popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "ambari-sudo.sh PATH=/bin -H -E usermod -s /bin/bash -u 1 mapred"], shell=False, preexec_fn=preexec_fn, stderr=-2, stdout=-1, env={'PATH': '/bin'}, cwd=None, close_fds=True)
self.assertEqual(popen_mock.call_count, 1)
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ff3d66a/ambari-common/src/main/python/resource_management/core/base.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/base.py b/ambari-common/src/main/python/resource_management/core/base.py
index 1500e1f..b862853 100644
--- a/ambari-common/src/main/python/resource_management/core/base.py
+++ b/ambari-common/src/main/python/resource_management/core/base.py
@@ -58,6 +58,17 @@ class BooleanArgument(ResourceArgument):
"Expected a boolean for %s received %r" % (self.name, value))
return value
+class IntegerArgument(ResourceArgument):
+ def validate(self, value):
+ if value is None:
+ return value
+
+ value = super(IntegerArgument, self).validate(value)
+ if not isinstance( value, int ):
+ raise InvalidArgument(
+ "Expected an integer for %s received %r" % (self.name, value))
+ return value
+
class PasswordArgument(ResourceArgument):
def log_str(self, key, value):
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ff3d66a/ambari-common/src/main/python/resource_management/core/providers/accounts.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/providers/accounts.py b/ambari-common/src/main/python/resource_management/core/providers/accounts.py
index c4f2496..fa70989 100644
--- a/ambari-common/src/main/python/resource_management/core/providers/accounts.py
+++ b/ambari-common/src/main/python/resource_management/core/providers/accounts.py
@@ -66,7 +66,9 @@ class UserProvider(Provider):
groups = self.resource.groups
if self.user and self.user_groups:
groups += self.user_groups
- option_value = ",".join(groups)
+ option_value = ",".join(groups)
+ elif attributes[1] == "-u" and self.user and self.user.pw_uid == getattr(self.resource, option_name):
+ option_value = None
else:
option_value = getattr(self.resource, option_name)
@@ -78,7 +80,7 @@ class UserProvider(Provider):
return
command.append(self.resource.username)
-
+
shell.checked_call(command, sudo=True)
def action_remove(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ff3d66a/ambari-common/src/main/python/resource_management/core/resources/accounts.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/resources/accounts.py b/ambari-common/src/main/python/resource_management/core/resources/accounts.py
index 4ee2c57..fb3f35a 100644
--- a/ambari-common/src/main/python/resource_management/core/resources/accounts.py
+++ b/ambari-common/src/main/python/resource_management/core/resources/accounts.py
@@ -21,7 +21,7 @@ Ambari Agent
"""
__all__ = ["Group", "User"]
-from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
+from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument, IntegerArgument
class Group(Resource):
@@ -37,7 +37,7 @@ class User(Resource):
action = ForcedListArgument(default="create")
username = ResourceArgument(default=lambda obj: obj.name)
comment = ResourceArgument()
- uid = ResourceArgument()
+ uid = IntegerArgument()
gid = ResourceArgument()
"""
If the user exists, and there are some groups, appends to existant
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ff3d66a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
index 886bc45..aa33261 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
@@ -49,11 +49,13 @@ def setup_users():
uid = get_uid(user),
gid = params.user_to_gid_dict[user],
groups = params.user_to_groups_dict[user],
+ fetch_nonlocal_groups = params.fetch_nonlocal_groups,
)
else:
User(user,
gid = params.user_to_gid_dict[user],
groups = params.user_to_groups_dict[user],
+ fetch_nonlocal_groups = params.fetch_nonlocal_groups,
)
if params.override_uid == "true":
@@ -96,7 +98,7 @@ def create_dfs_cluster_admins():
User(params.hdfs_user,
groups = params.user_to_groups_dict[params.hdfs_user] + groups_list,
- fetch_nonlocal_groups = params.fetch_nonlocal_groups
+ fetch_nonlocal_groups = params.fetch_nonlocal_groups
)
def create_tez_am_view_acls():
@@ -145,7 +147,7 @@ def set_uid(user, user_dirs):
mode=0555)
ignore_groupsusers_create_str = str(params.ignore_groupsusers_create).lower()
uid = get_uid(user)
- Execute(format("{tmp_dir}/changeUid.sh {user} {user_dirs} {uid}"),
+ Execute(format("{tmp_dir}/changeUid.sh {user} {user_dirs} {new_uid}", new_uid=0 if uid is None else uid),
not_if = format("(test $(id -u {user}) -gt 1000) || ({ignore_groupsusers_create_str})"))
def get_uid(user):
@@ -161,12 +163,12 @@ def get_uid(user):
return uid
else:
if user == params.smoke_user:
- return 0
+ return None
File(format("{tmp_dir}/changeUid.sh"),
content=StaticFile("changeToSecureUid.sh"),
mode=0555)
- conde, newUid = shell.call((format("{tmp_dir}/changeUid.sh"), format("{user}")), sudo=True)
- return newUid
+ code, newUid = shell.call(format("{tmp_dir}/changeUid.sh {user}"))
+ return int(newUid)
def setup_hadoop_env():
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ff3d66a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
index 1d2351f..a13ac24 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
@@ -58,8 +58,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'hive',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'hadoop'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -67,8 +68,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'oozie',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'users'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -76,13 +78,15 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'nobody',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'nobody'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('User', 'ambari-qa',
gid = 'hadoop',
- uid = 0,
+ uid = None,
groups = [u'users'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -90,8 +94,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'flume',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'hadoop'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -99,8 +104,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'hdfs',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'hadoop'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -108,8 +114,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'storm',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'hadoop'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -117,8 +124,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'mapred',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'hadoop'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -126,8 +134,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'hbase',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'hadoop'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -135,8 +144,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'tez',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'users'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -144,8 +154,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'zookeeper',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'hadoop'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -153,8 +164,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'falcon',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'users'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -162,8 +174,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'sqoop',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'hadoop'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -171,8 +184,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'yarn',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'hadoop'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
@@ -180,8 +194,9 @@ class TestHookBeforeInstall(RMFTestCase):
)
self.assertResourceCalled('User', 'hcat',
gid = 'hadoop',
- uid = '1000',
+ uid = 1000,
groups = [u'hadoop'],
+ fetch_nonlocal_groups = True,
)
self.assertResourceCalled('File', '/tmp/changeUid.sh',
content = StaticFile('changeToSecureUid.sh'),
[18/23] ambari git commit: AMBARI-21533 Text change for bypassing
prechecks. (atkach)
Posted by jo...@apache.org.
AMBARI-21533 Text change for bypassing prechecks. (atkach)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f9ce03e1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f9ce03e1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f9ce03e1
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: f9ce03e161c0b5c1b16c6d5d337913b743182276
Parents: 9ff3d66
Author: Andrii Tkach <at...@apache.org>
Authored: Thu Jul 20 13:40:28 2017 +0300
Committer: Andrii Tkach <at...@apache.org>
Committed: Thu Jul 20 13:40:28 2017 +0300
----------------------------------------------------------------------
ambari-web/app/messages.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9ce03e1/ambari-web/app/messages.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js
index 9ec64ef..1bad287 100644
--- a/ambari-web/app/messages.js
+++ b/ambari-web/app/messages.js
@@ -1744,7 +1744,7 @@ Em.I18n.translations = {
'admin.stackVersions.version.upgrade.upgradeOptions.EU.title': "Express Upgrade",
'admin.stackVersions.version.upgrade.upgradeOptions.HOU.title': "Host Ordered Upgrade",
'admin.stackVersions.version.upgrade.upgradeOptions.EU.description': "Services are stopped while the upgrade is performed. Incurs downtime, but faster upgrade.",
- 'admin.stackVersions.version.upgrade.upgradeOptions.errors_bypassed': "Bypassed errors,<br/>proceed at your own risk.",
+ 'admin.stackVersions.version.upgrade.upgradeOptions.errors_bypassed': "Upgrade Checks Bypassed.",
'admin.stackVersions.version.upgrade.upgradeOptions.preCheck.rerun':'Rerun Checks',
'admin.stackVersions.version.upgrade.upgradeOptions.preCheck.msg.title':'Checks:',
'admin.stackVersions.version.upgrade.upgradeOptions.preCheck.msg.checking': 'Checking...',
[09/23] ambari git commit: AMBARI-21520. Ambari server logs NPE with
no additional stack trace on any host component start/stop
command.(vbrodetskyi)
Posted by jo...@apache.org.
AMBARI-21520. Ambari server logs NPE with no additional stack trace on any host component start/stop command.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2b24cde4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2b24cde4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2b24cde4
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 2b24cde4315212a782caa88294d98e3338c86ce4
Parents: 7542bc3
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Wed Jul 19 16:38:13 2017 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Wed Jul 19 16:38:13 2017 +0300
----------------------------------------------------------------------
.../AmbariManagementControllerImpl.java | 4 +++
.../ambari/server/state/ConfigHelper.java | 29 ++++++++++++++++++--
2 files changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b24cde4/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index b80ee98..cccb4b2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2727,6 +2727,10 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
return requestStages;
}
+ // check all stack configs are present in desired configs
+ configHelper.checkAllStageConfigsPresentInDesiredConfigs(cluster);
+
+
// caching effective cluster version
ClusterVersionEntity effectiveClusterVersion = cluster.getEffectiveClusterVersion();
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b24cde4/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index a4c5511..6e60890 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -698,7 +698,9 @@ public class ConfigHelper {
for (PropertyInfo stackProperty : stackProperties) {
if (stackProperty.getPropertyTypes().contains(propertyType)) {
String stackPropertyConfigType = fileNameToConfigType(stackProperty.getFilename());
- result.put(stackProperty, actualConfigs.get(stackPropertyConfigType).getProperties().get(stackProperty.getName()));
+ if (actualConfigs.containsKey(stackPropertyConfigType)) {
+ result.put(stackProperty, actualConfigs.get(stackPropertyConfigType).getProperties().get(stackProperty.getName()));
+ }
}
}
@@ -764,13 +766,36 @@ public class ConfigHelper {
for (PropertyInfo stackProperty : stackProperties) {
if (stackProperty.getPropertyTypes().contains(propertyType)) {
String stackPropertyConfigType = fileNameToConfigType(stackProperty.getFilename());
- result.add(actualConfigs.get(stackPropertyConfigType).getProperties().get(stackProperty.getName()));
+ if (actualConfigs.containsKey(stackPropertyConfigType)) {
+ result.add(actualConfigs.get(stackPropertyConfigType).getProperties().get(stackProperty.getName()));
+ }
}
}
return result;
}
+ public void checkAllStageConfigsPresentInDesiredConfigs(Cluster cluster) throws AmbariException {
+ StackId stackId = cluster.getDesiredStackVersion();
+ Set<String> stackConfigTypes = ambariMetaInfo.getStack(stackId.getStackName(),
+ stackId.getStackVersion()).getConfigTypeAttributes().keySet();
+ Map<String, Config> actualConfigs = new HashMap<>();
+ Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs();
+
+ for (Map.Entry<String, DesiredConfig> desiredConfigEntry : desiredConfigs.entrySet()) {
+ String configType = desiredConfigEntry.getKey();
+ DesiredConfig desiredConfig = desiredConfigEntry.getValue();
+ actualConfigs.put(configType, cluster.getConfig(configType, desiredConfig.getTag()));
+ }
+
+ for (String stackConfigType : stackConfigTypes) {
+ if (!actualConfigs.containsKey(stackConfigType)) {
+ LOG.error(String.format("Unable to find stack configuration %s in ambari configs!", stackConfigType));
+ }
+ }
+
+ }
+
/***
* Fetch all the config values of a given PropertyType. For eg: Fetch all stack configs that are of type "user"
* @param cluster
[03/23] ambari git commit: AMBARI-21487. Add SOLR to BigInsights
4.2.5 stack (oleewere)
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr-client-log4j.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr-client-log4j.properties.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr-client-log4j.properties.j2
new file mode 100644
index 0000000..1ac7803
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr-client-log4j.properties.j2
@@ -0,0 +1,38 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO,file,stdout,stderr
+
+log4j.appender.file=org.apache.log4j.RollingFileAppender
+log4j.appender.file.File={{solr_client_log|default('/var/log/solr-client/solr-client.log')}}
+log4j.appender.file.MaxFileSize=80MB
+log4j.appender.file.MaxBackupIndex=60
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Threshold=INFO
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%m%n
+
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Threshold=ERROR
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr-env.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr-env.sh.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr-env.sh.j2
new file mode 100644
index 0000000..24b93ca
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr-env.sh.j2
@@ -0,0 +1,126 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# By default the script will use JAVA_HOME to determine which java
+# to use, but you can set a specific path for Solr to use without
+# affecting other Java applications on your server/workstation.
+SOLR_JAVA_HOME={{java64_home}}
+
+# Increase Java Min/Max Heap as needed to support your indexing / query needs
+SOLR_JAVA_MEM="-Xms{{solr_min_mem}}m -Xmx{{solr_max_mem}}m"
+
+# Enable verbose GC logging
+GC_LOG_OPTS="-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \
+-XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime"
+
+# These GC settings have shown to work well for a number of common Solr workloads
+GC_TUNE="-XX:NewRatio=3 \
+-XX:SurvivorRatio=4 \
+-XX:TargetSurvivorRatio=90 \
+-XX:MaxTenuringThreshold=8 \
+-XX:+UseConcMarkSweepGC \
+-XX:+UseParNewGC \
+-XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \
+-XX:+CMSScavengeBeforeRemark \
+-XX:PretenureSizeThreshold=64m \
+-XX:+UseCMSInitiatingOccupancyOnly \
+-XX:CMSInitiatingOccupancyFraction=50 \
+-XX:CMSMaxAbortablePrecleanTime=6000 \
+-XX:+CMSParallelRemarkEnabled \
+-XX:+ParallelRefProcEnabled"
+
+# Set the ZooKeeper connection string if using an external ZooKeeper ensemble
+# e.g. host1:2181,host2:2181/chroot
+# Leave empty if not using SolrCloud
+ZK_HOST="{{zookeeper_quorum}}{{solr_znode}}"
+
+# Set the ZooKeeper client timeout (for SolrCloud mode)
+ZK_CLIENT_TIMEOUT="60000"
+
+# By default the start script uses "localhost"; override the hostname here
+# for production SolrCloud environments to control the hostname exposed to cluster state
+#SOLR_HOST="192.168.1.1"
+
+# By default the start script uses UTC; override the timezone if needed
+#SOLR_TIMEZONE="UTC"
+
+# Set to true to activate the JMX RMI connector to allow remote JMX client applications
+# to monitor the JVM hosting Solr; set to "false" to disable that behavior
+# (false is recommended in production environments)
+ENABLE_REMOTE_JMX_OPTS="true"
+
+# The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here
+RMI_PORT={{solr_jmx_port}}
+
+# Anything you add to the SOLR_OPTS variable will be included in the java
+# start command line as-is, in ADDITION to other options. If you specify the
+# -a option on start script, those options will be appended as well. Examples:
+#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000"
+#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoCommit.maxTime=60000"
+#SOLR_OPTS="$SOLR_OPTS -Dsolr.clustering.enabled=true"
+
+# Location where the bin/solr script will save PID files for running instances
+# If not set, the script will create PID files in $SOLR_TIP/bin
+SOLR_PID_DIR={{solr_piddir}}
+
+# Path to a directory where Solr creates index files, the specified directory
+# must contain a solr.xml; by default, Solr will use server/solr
+SOLR_HOME={{solr_datadir}}
+
+# Solr provides a default Log4J configuration properties file in server/resources
+# however, you may want to customize the log settings and file appender location
+# so you can point the script to use a different log4j.properties file
+LOG4J_PROPS={{solr_conf}}/log4j.properties
+
+# Location where Solr should write logs to; should agree with the file appender
+# settings in server/resources/log4j.properties
+SOLR_LOGS_DIR={{solr_log_dir}}
+
+# Sets the port Solr binds to, default is 8983
+SOLR_PORT={{solr_port}}
+
+# Be sure to update the paths to the correct keystore for your environment
+{% if solr_ssl_enabled %}
+SOLR_SSL_KEY_STORE={{solr_keystore_location}}
+SOLR_SSL_KEY_STORE_PASSWORD={{solr_keystore_password}}
+SOLR_SSL_TRUST_STORE={{solr_keystore_location}}
+SOLR_SSL_TRUST_STORE_PASSWORD={{solr_keystore_password}}
+SOLR_SSL_NEED_CLIENT_AUTH=false
+SOLR_SSL_WANT_CLIENT_AUTH=false
+{% endif %}
+
+{% if security_enabled -%}
+SOLR_HOST=`hostname -f`
+SOLR_JAAS_FILE={{solr_jaas_file}}
+SOLR_KERB_KEYTAB={{solr_web_kerberos_keytab}}
+SOLR_KERB_PRINCIPAL={{solr_web_kerberos_principal}}
+SOLR_KERB_NAME_RULES={{solr_kerberos_name_rules}}
+
+SOLR_AUTHENTICATION_CLIENT_CONFIGURER="org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer"
+SOLR_AUTHENTICATION_OPTS=" -DauthenticationPlugin=org.apache.solr.security.KerberosPlugin -Djava.security.auth.login.config=$SOLR_JAAS_FILE -Dsolr.kerberos.principal=${SOLR_KERB_PRINCIPAL} -Dsolr.kerberos.keytab=${SOLR_KERB_KEYTAB} -Dsolr.kerberos.cookie.domain=${SOLR_HOST} -Dsolr.kerberos.name.rules=${SOLR_KERB_NAME_RULES}"
+{% endif %}
+
+# Comment out the following SOLR_OPTS setting to config Solr to write its index and transaction log files to local filesystem.
+# Data (index and transaction log files) exists on HDFS will not be moved to local filesystem,
+# after you change this config, they will not be available from local filesystem.
+SOLR_OPTS="-Dsolr.directoryFactory=HdfsDirectoryFactory \
+-Dsolr.lock.type=hdfs \
+-Dsolr.hdfs.confdir=/etc/hadoop/conf \
+-Dsolr.hdfs.home={{fs_root}}{{solr_hdfs_home_dir}} \
+-Dsolr.hdfs.security.kerberos.enabled={{security_enabled}} \
+-Dsolr.hdfs.security.kerberos.keytabfile={{solr_kerberos_keytab}} \
+-Dsolr.hdfs.security.kerberos.principal={{solr_kerberos_principal}} \
+-Dsolr.log4j.dir={{solr_log_dir}}"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr-log4j.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr-log4j.properties.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr-log4j.properties.j2
new file mode 100644
index 0000000..4159a59
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr-log4j.properties.j2
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Logging level
+solr.log.dir={{solr_log_dir}}
+#log4j.rootLogger=INFO, file, CONSOLE
+log4j.rootLogger=WARN, file
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x [%X{collection} %X{shard} %X{replica} %X{core}] \u2013 %m%n
+
+#- size rotation with log cleanup.
+log4j.appender.file=org.apache.log4j.RollingFileAppender
+log4j.appender.file.MaxFileSize=10MB
+log4j.appender.file.MaxBackupIndex=9
+
+#- File to log to and log format
+log4j.appender.file.File=${solr.log.dir}/solr.log
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{ISO8601} [%t] %-5p [%X{collection} %X{shard} %X{replica} %X{core}] %C (%F:%L) - %m%n
+
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.hadoop=WARN
+
+# set to INFO to enable infostream log messages
+log4j.logger.org.apache.solr.update.LoggingInfoStream=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr.xml.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr.xml.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr.xml.j2
new file mode 100644
index 0000000..45f9c13
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/properties/solr.xml.j2
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<solr>
+ <solrcloud>
+ <str name="host">${host:}</str>
+ <int name="hostPort">${jetty.port:}</int>
+ <str name="hostContext">${hostContext:solr}</str>
+ <int name="zkClientTimeout">${zkClientTimeout:15000}</int>
+ <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
+ </solrcloud>
+</solr>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/quicklinks/quicklinks.json b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/quicklinks/quicklinks.json
new file mode 100644
index 0000000..e7690b8
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/quicklinks/quicklinks.json
@@ -0,0 +1,37 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol": {
+ "type":"https",
+ "checks":[
+ {
+ "property":"solr_ssl_enabled",
+ "desired":"true",
+ "site":"solr-env"
+ }
+ ]
+ },
+ "links": [
+ {
+ "name": "solr_ui",
+ "label": "Solr Admin UI",
+ "component_name": "SOLR",
+ "url": "%@://%@:%@",
+ "requires_user_name": "false",
+ "port": {
+ "http_property": "solr_port",
+ "http_default_port": "8983",
+ "https_property": "solr_port",
+ "https_default_port": "8983",
+ "regex": "^(\\d+)$",
+ "site": "solr-env"
+ },
+ "knox_url": "%@://%@:%@/%@",
+ "knox_path": "gateway/solrui/solr/",
+ "supports_knox": "true"
+ }
+ ]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/512caee1/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/themes/theme.json b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/themes/theme.json
new file mode 100644
index 0000000..3b076fb
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/SOLR/themes/theme.json
@@ -0,0 +1,107 @@
+{
+ "name": "default",
+ "description": "Default theme for SOLR service",
+ "configuration": {
+ "layouts": [
+ {
+ "name": "default",
+ "tabs": [
+ {
+ "name": "settings",
+ "display-name": "Settings",
+ "layout": {
+ "tab-columns": "2",
+ "tab-rows": "1",
+ "sections": [
+ {
+ "name": "section-solr",
+ "display-name": "Solr",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "4",
+ "subsections": [
+ {
+ "name": "subsection-solr-col1",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ },
+ {
+ "name": "subsection-solr-col2",
+ "row-index": "0",
+ "column-index": "1",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "solr-env/solr_minmem",
+ "subsection-name": "subsection-solr-col1"
+ },
+ {
+ "config": "solr-env/solr_maxmem",
+ "subsection-name": "subsection-solr-col1"
+ },
+ {
+ "config": "solr-env/solr_datadir",
+ "subsection-name": "subsection-solr-col2"
+ },
+ {
+ "config": "solr-env/solr_znode",
+ "subsection-name": "subsection-solr-col2"
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "solr-env/solr_minmem",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "GB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "solr-env/solr_maxmem",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "GB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "solr-env/solr_datadir",
+ "widget": {
+ "type": "directory"
+ }
+ },
+ {
+ "config": "solr-env/solr_znode",
+ "widget": {
+ "type": "directory"
+ }
+ }
+ ]
+ }
+}
+