You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by yu...@apache.org on 2015/12/16 21:26:23 UTC
[1/2] ambari git commit: AMBARI-14390. EU from HDP 2.1 to 2.3 added
NULL properties (afernandez via dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/branch-rbac-sso 5ffde649e -> 0d33a072e
AMBARI-14390. EU from HDP 2.1 to 2.3 added NULL properties (afernandez via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5a968537
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5a968537
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5a968537
Branch: refs/heads/branch-rbac-sso
Commit: 5a96853727f159c7b73fed71b03a7a5656d94fce
Parents: 5ffde64
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Wed Dec 16 10:00:27 2015 +0200
Committer: Yusaku Sako <yu...@hortonworks.com>
Committed: Wed Dec 16 12:25:33 2015 -0800
----------------------------------------------------------------------
.../internal/UpgradeResourceProvider.java | 11 +++++++
.../ambari/server/state/ConfigMergeHelper.java | 2 ++
.../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml | 14 ++++++++
.../stacks/HDP/2.2/upgrades/config-upgrade.xml | 34 ++++++++++++++++++++
4 files changed, 61 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a968537/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 1e59b58..0f9e65e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -1026,6 +1026,17 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
if (null == newDefaultConfigurations) {
newConfigurationsByType.put(configurationType, existingConfigurations);
continue;
+ } else {
+ // TODO, should we remove existing configs whose value is NULL even though they don't have a value in the new stack?
+
+ // Remove any configs in the new stack whose value is NULL, unless they currently exist and the value is not NULL.
+ Iterator<Map.Entry<String, String>> iter = newDefaultConfigurations.entrySet().iterator();
+ while (iter.hasNext()) {
+ Map.Entry<String, String> entry = iter.next();
+ if (entry.getValue() == null) {
+ iter.remove();
+ }
+ }
}
// for every existing configuration, see if an entry exists; if it does
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a968537/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigMergeHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigMergeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigMergeHelper.java
index b799bc7..9d366ae 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigMergeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigMergeHelper.java
@@ -78,6 +78,7 @@ public class ConfigMergeHelper {
targetStack.getStackName(), targetStack.getStackVersion());
addToMap(newMap, set);
+ // Final result after merging.
Map<String, Map<String, ThreeWayValue>> result =
new HashMap<String, Map<String, ThreeWayValue>>();
@@ -104,6 +105,7 @@ public class ConfigMergeHelper {
customValueKeys = CollectionUtils.subtract(valueKeys, oldPairs.keySet());
}
+ // Keep properties with custom values (i.e., changed from default value in old stack)
if (null != customValueKeys) {
for (String prop : customValueKeys) {
String newVal = newPairs.get(prop);
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a968537/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
index fff9c97..2ff4662 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
@@ -221,11 +221,25 @@
<supports-auto-skip-failure>false</supports-auto-skip-failure>
<!--Changes for stack 2.2-->
+ <!--HDFS-->
+ <execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode without Ranger">
+ <task xsi:type="configure" id="hdp_2_2_0_0_namenode_no_ranger"/>
+ </execute-stage>
+
<!--YARN-->
<execute-stage service="MAPREDUCE2" component="HISTORYSERVER" title="Apply config changes for HistoryServer">
<task xsi:type="configure" id="hdp_2_2_0_0_historyserver_classpath"/>
</execute-stage>
+ <execute-stage service="MAPREDUCE2" component="HISTORYSERVER" title="Apply config changes for YARN without Ranger">
+ <task xsi:type="configure" id="hdp_2_2_0_0_historyserver_no_ranger"/>
+ </execute-stage>
+
+ <!--STORM-->
+ <execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus without Ranger">
+ <task xsi:type="configure" id="hdp_2_2_0_0_nimbus_no_ranger"/>
+ </execute-stage>
+
<!--Changes for stack 2.3-->
<!--HDFS-->
<execute-stage service="HDFS" component="NAMENODE" title="Apply config changes for NameNode">
http://git-wip-us.apache.org/repos/asf/ambari/blob/5a968537/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
index ab384db..c2844fa 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
@@ -19,6 +19,20 @@
<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<services>
+
+ <service name="HDFS">
+ <component name="NAMENODE">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_2_0_0_namenode_no_ranger" summary="NameNode properties without Ranger">
+ <!-- In HDP 2.1, RANGER was not present, so this property must not be added even though it is new in the
+ HDP 2.2 stack. The property is only valid in HDP 2.2 and higher if RANGER is present. -->
+ <type>hdfs-site</type>
+ <transfer operation="delete" delete-key="dfs.namenode.inode.attributes.provider.class"/>
+ </definition>
+ </changes>
+ </component>
+ </service>
+
<service name="YARN">
<component name="HISTORY_SERVER">
<changes>
@@ -26,6 +40,13 @@
<type>yarn-site</type>
<set key="yarn.application.classpath" value="$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*"/>
</definition>
+
+ <definition xsi:type="configure" id="hdp_2_2_0_0_historyserver_no_ranger" summary="YARN properties without Ranger">
+ <!-- In HDP 2.1, RANGER was not present, so this property must not be added even though it is new in the
+ HDP 2.2 stack. The property is only valid in HDP 2.2 and higher if RANGER is present. -->
+ <type>yarn-site</type>
+ <transfer operation="delete" delete-key="yarn.authorization-provider"/>
+ </definition>
</changes>
</component>
</service>
@@ -91,4 +112,17 @@
</service>
</services>
+ <service name="STORM">
+ <component name="NIMBUS">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_2_0_0_nimbus_no_ranger" summary="Nimbus properties without Ranger">
+ <!-- In HDP 2.1, RANGER was not present, so this property must not be added even though it is new in the
+ HDP 2.2 stack. The property is only valid in HDP 2.2 and higher if RANGER is present. -->
+ <type>storm-site</type>
+ <transfer operation="delete" delete-key="nimbus.authorizer"/>
+ </definition>
+ </changes>
+ </component>
+ </service>
+
</upgrade-config-changes>
[2/2] ambari git commit: AMBARI-14400. Spark server restart failed
(java.lang.NumberFormatException) causing express upgrade to fail. (aonishuk)
Posted by yu...@apache.org.
AMBARI-14400. Spark server restart failed (java.lang.NumberFormatException) causing express upgrade to fail. (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0d33a072
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0d33a072
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0d33a072
Branch: refs/heads/branch-rbac-sso
Commit: 0d33a072e9ce4aa65100b8d530d4118dc0db17d8
Parents: 5a96853
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Wed Dec 16 20:10:26 2015 +0200
Committer: Yusaku Sako <yu...@hortonworks.com>
Committed: Wed Dec 16 12:26:13 2015 -0800
----------------------------------------------------------------------
.../common-services/SPARK/1.2.0.2.2/package/scripts/params.py | 3 +--
.../services/SPARK/configuration/spark-hive-site-override.xml | 7 +++++++
2 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0d33a072/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
index 84ad34d..5e2c10c 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
@@ -103,7 +103,7 @@ spark_jobhistoryserver_hosts = default("/clusterHostInfo/spark_jobhistoryserver_
if len(spark_jobhistoryserver_hosts) > 0:
spark_history_server_host = spark_jobhistoryserver_hosts[0]
-else:
+else:SPARK/1.2.0.2.2/package/scripts/params.py
spark_history_server_host = "localhost"
# spark-defaults params
@@ -160,7 +160,6 @@ if has_spark_thriftserver and 'spark-thrift-sparkconf' in config['configurations
# update default metastore client properties (async wait for metastore component) it is useful in case of
# blueprint provisioning when hive-metastore and spark-thriftserver is not on the same host.
spark_hive_properties.update({
- 'hive.metastore.client.connect.retry.delay' : config['configurations']['hive-site']['hive.metastore.client.connect.retry.delay'],
'hive.metastore.connect.retries' : config['configurations']['hive-site']['hive.metastore.connect.retries'],
'hive.metastore.client.socket.timeout' : config['configurations']['hive-site']['hive.metastore.client.socket.timeout']
})
http://git-wip-us.apache.org/repos/asf/ambari/blob/0d33a072/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-hive-site-override.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-hive-site-override.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-hive-site-override.xml
index 54df516..d65c8b1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-hive-site-override.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-hive-site-override.xml
@@ -19,6 +19,13 @@ limitations under the License.
<configuration supports_final="true">
<property>
+ <name>hive.metastore.client.connect.retry.delay</name>
+ <value>5</value>
+ <description>
+ Expects a time value - number of seconds for the client to wait between consecutive connection attempts
+ </description>
+ </property>
+ <property>
<name>hive.server2.thrift.port</name>
<value>10015</value>
<description>