You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2016/03/21 19:45:26 UTC
[1/3] ambari git commit: AMBARI-15420: Refactor resource_management
library (jluniya)
Repository: ambari
Updated Branches:
refs/heads/trunk aa8409fd1 -> 0f3bf8aa8
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/test/python/stacks/2.1/TEZ/test_tez_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/TEZ/test_tez_client.py b/ambari-server/src/test/python/stacks/2.1/TEZ/test_tez_client.py
index ab08776..e53eb4b 100644
--- a/ambari-server/src/test/python/stacks/2.1/TEZ/test_tez_client.py
+++ b/ambari-server/src/test/python/stacks/2.1/TEZ/test_tez_client.py
@@ -75,7 +75,7 @@ class TestTezClient(RMFTestCase):
get_stack_version_mock.return_value = "2.2.1.0-2067"
self.assertResourceCalled("Execute", ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hadoop-client', '2.2.1.0-2067'), sudo=True)
- # for now, it's enough that hdp-select is confirmed
+ # for now, it's enough that <stack-selector-tool> is confirmed
@patch("resource_management.libraries.functions.get_stack_version")
def test_upgrade_23(self, get_stack_version_mock):
@@ -89,7 +89,7 @@ class TestTezClient(RMFTestCase):
get_stack_version_mock.return_value = "2.2.1.0-2067"
self.assertResourceCalled("Execute", ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hadoop-client', '2.2.1.0-2067'), sudo=True)
- # for now, it's enough that hdp-select is confirmed
+ # for now, it's enough that <stack-selector-tool> is confirmed
def test_pre_upgrade_restart_23(self):
config_file = self.get_src_folder()+"/test/python/stacks/2.1/configs/client-upgrade.json"
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/contrib/views/capacity-scheduler/src/main/resources/view.log4j.properties
----------------------------------------------------------------------
diff --git a/contrib/views/capacity-scheduler/src/main/resources/view.log4j.properties b/contrib/views/capacity-scheduler/src/main/resources/view.log4j.properties
index 1c51fb5..f10b694 100644
--- a/contrib/views/capacity-scheduler/src/main/resources/view.log4j.properties
+++ b/contrib/views/capacity-scheduler/src/main/resources/view.log4j.properties
@@ -1,3 +1,21 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
log4j.appender.capshedView=org.apache.log4j.RollingFileAppender
log4j.appender.capshedView.File=/var/log/ambari-server/capshed-view.log
log4j.appender.capshedView.MaxFileSize=80MB
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/contrib/views/files/src/main/resources/view.log4j.properties
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/main/resources/view.log4j.properties b/contrib/views/files/src/main/resources/view.log4j.properties
index 4340341..18c9d32 100644
--- a/contrib/views/files/src/main/resources/view.log4j.properties
+++ b/contrib/views/files/src/main/resources/view.log4j.properties
@@ -1,3 +1,21 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
log4j.appender.filesView=org.apache.log4j.RollingFileAppender
log4j.appender.filesView.File=/var/log/ambari-server/files-view.log
log4j.appender.filesView.MaxFileSize=80MB
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/contrib/views/hive/src/main/resources/view.log4j.properties
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/resources/view.log4j.properties b/contrib/views/hive/src/main/resources/view.log4j.properties
index 77f8258..b0c61c3 100644
--- a/contrib/views/hive/src/main/resources/view.log4j.properties
+++ b/contrib/views/hive/src/main/resources/view.log4j.properties
@@ -1,3 +1,20 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
log4j.appender.hiveView=org.apache.log4j.RollingFileAppender
log4j.appender.hiveView.File=/var/log/ambari-server/hive-view.log
log4j.appender.hiveView.MaxFileSize=80MB
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/contrib/views/jobs/src/main/resources/view.log4j.properties
----------------------------------------------------------------------
diff --git a/contrib/views/jobs/src/main/resources/view.log4j.properties b/contrib/views/jobs/src/main/resources/view.log4j.properties
index 4007e0e..27c49c5 100644
--- a/contrib/views/jobs/src/main/resources/view.log4j.properties
+++ b/contrib/views/jobs/src/main/resources/view.log4j.properties
@@ -1,3 +1,21 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
log4j.appender.jobsView=org.apache.log4j.RollingFileAppender
log4j.appender.jobsView.File=/var/log/ambari-server/jobs-view.log
log4j.appender.jobsView.MaxFileSize=80MB
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/contrib/views/pig/src/main/resources/view.log4j.properties
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/resources/view.log4j.properties b/contrib/views/pig/src/main/resources/view.log4j.properties
index 38eb5ee..50c19c4 100644
--- a/contrib/views/pig/src/main/resources/view.log4j.properties
+++ b/contrib/views/pig/src/main/resources/view.log4j.properties
@@ -1,3 +1,21 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
log4j.appender.pigView=org.apache.log4j.RollingFileAppender
log4j.appender.pigView.File=/var/log/ambari-server/pig-view.log
log4j.appender.pigView.MaxFileSize=80MB
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/contrib/views/slider/src/main/resources/view.log4j.properties
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/resources/view.log4j.properties b/contrib/views/slider/src/main/resources/view.log4j.properties
index 2eed2ca..1ee9483 100644
--- a/contrib/views/slider/src/main/resources/view.log4j.properties
+++ b/contrib/views/slider/src/main/resources/view.log4j.properties
@@ -1,3 +1,21 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
log4j.appender.sliderView=org.apache.log4j.RollingFileAppender
log4j.appender.sliderView.File=/var/log/ambari-server/slider-view.log
log4j.appender.sliderView.MaxFileSize=80MB
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/contrib/views/storm/src/main/resources/view.log4j.properties
----------------------------------------------------------------------
diff --git a/contrib/views/storm/src/main/resources/view.log4j.properties b/contrib/views/storm/src/main/resources/view.log4j.properties
index a51fb74..c62ca30 100644
--- a/contrib/views/storm/src/main/resources/view.log4j.properties
+++ b/contrib/views/storm/src/main/resources/view.log4j.properties
@@ -1,3 +1,21 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
log4j.appender.stormView=org.apache.log4j.RollingFileAppender
log4j.appender.stormView.File=/var/log/ambari-server/storm-view.log
log4j.appender.stormView.MaxFileSize=80MB
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/contrib/views/tez/src/main/resources/view.log4j.properties
----------------------------------------------------------------------
diff --git a/contrib/views/tez/src/main/resources/view.log4j.properties b/contrib/views/tez/src/main/resources/view.log4j.properties
index c7cf0be..25d16d0 100644
--- a/contrib/views/tez/src/main/resources/view.log4j.properties
+++ b/contrib/views/tez/src/main/resources/view.log4j.properties
@@ -1,3 +1,21 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
log4j.appender.tezView=org.apache.log4j.RollingFileAppender
log4j.appender.tezView.File=/var/log/ambari-server/tez-view.log
log4j.appender.tezView.MaxFileSize=80MB
[2/3] ambari git commit: AMBARI-15420: Refactor resource_management
library (jluniya)
Posted by jl...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
index 4be4049..434aa96 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
@@ -93,6 +93,7 @@ import static org.apache.ambari.server.controller.utilities.PropertyHelper.AGGRE
@Singleton
public class AmbariMetaInfo {
public static final String SERVICE_CONFIG_FOLDER_NAME = "configuration";
+ public static final String SERVICE_PROPERTIES_FOLDER_NAME = "properties";
public static final String SERVICE_THEMES_FOLDER_NAME = "themes";
public static final String SERVICE_QUICKLINKS_CONFIGURATIONS_FOLDER_NAME = "quicklinks";
public static final String SERVICE_CONFIG_FILE_NAME_POSTFIX = ".xml";
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/java/org/apache/ambari/server/stack/ConfigurationDirectory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ConfigurationDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ConfigurationDirectory.java
index 7f21aaa..84842e2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ConfigurationDirectory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ConfigurationDirectory.java
@@ -22,11 +22,15 @@ import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.stack.ConfigurationXml;
+import org.apache.ambari.server.utils.JsonUtils;
+import org.apache.commons.io.FileUtils;
+import org.eclipse.jetty.util.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.xml.namespace.QName;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -57,12 +61,20 @@ public class ConfigurationDirectory extends StackDefinitionDirectory {
private final static Logger LOG = LoggerFactory.getLogger(ConfigurationDirectory.class);
/**
+ * Properties directory
+ */
+ protected File propertiesDirFile;
+
+ /**
* Constructor.
*
* @param directoryName configuration directory name
*/
- public ConfigurationDirectory(String directoryName) {
+ public ConfigurationDirectory(String directoryName, String propertiesDirectoryName) {
super(directoryName);
+ if(!StringUtil.isBlank(propertiesDirectoryName)) {
+ propertiesDirFile = new File(propertiesDirectoryName);
+ }
parsePath();
}
@@ -126,6 +138,36 @@ public class ConfigurationDirectory extends StackDefinitionDirectory {
List<PropertyInfo> props = new ArrayList<PropertyInfo>();
for (PropertyInfo pi : configuration.getProperties()) {
pi.setFilename(fileName);
+ if(pi.getPropertyTypes().contains(PropertyInfo.PropertyType.VALUE_FROM_PROPERTY_FILE)) {
+ if(propertiesDirFile != null && propertiesDirFile.exists() && propertiesDirFile.isDirectory() ) {
+ String propertyFileName = pi.getPropertyValueAttributes().getPropertyFileName();
+ String propertyFileType = pi.getPropertyValueAttributes().getPropertyFileType();
+ String propertyFilePath = propertiesDirFile.getAbsolutePath() + File.separator + propertyFileName;
+ File propertyFile = new File(propertyFilePath);
+ if (propertyFile.exists() && propertyFile.isFile()) {
+ try {
+ String propertyValue = FileUtils.readFileToString(propertyFile);
+ switch (propertyFileType.toLowerCase()) {
+ case "json":
+ if(!JsonUtils.isValidJson(propertyValue)) {
+ LOG.error("Failed to load value from property file. Property file {} is not a valid JSON file", propertyFilePath);
+ break;
+ }
+ case "text":
+ default:
+ pi.setValue(propertyValue);
+ break;
+ }
+ } catch (IOException e) {
+ LOG.error("Failed to load value from property file {}. Error Message {}", propertyFilePath, e.getMessage());
+ }
+ } else {
+ LOG.error("Failed to load value from property file. Properties file {} does not exist", propertyFilePath);
+ }
+ } else {
+ LOG.error("Failed to load value from property file. Properties directory {} does not exist", propertiesDirFile.getAbsolutePath());
+ }
+ }
props.add(pi);
}
return props; }
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
index 0c7faea..b7e09a9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
@@ -328,7 +328,7 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem
*/
private void populateConfigurationModules() {
ConfigurationDirectory configDirectory = serviceDirectory.getConfigurationDirectory(
- serviceInfo.getConfigDir());
+ serviceInfo.getConfigDir(), AmbariMetaInfo.SERVICE_PROPERTIES_FOLDER_NAME);
if (configDirectory != null) {
for (ConfigurationModule config : configDirectory.getConfigurationModules()) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java
index c739211..f2e00fe 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java
@@ -61,11 +61,16 @@ public abstract class StackDefinitionDirectory {
* @param directoryName name of the configuration directory
* @return ConfigurationDirectory instance for the specified configuration directory name
*/
- public ConfigurationDirectory getConfigurationDirectory(String directoryName) {
+ public ConfigurationDirectory getConfigurationDirectory(String directoryName, String propertiesDirectoryName) {
ConfigurationDirectory configDirectory = null;
File configDirFile = new File(directory.getAbsolutePath() + File.separator + directoryName);
+ File propertiesDirFile = new File(directory.getAbsolutePath() + File.separator + propertiesDirectoryName);
if (configDirFile.exists() && configDirFile.isDirectory()) {
- configDirectory = new ConfigurationDirectory(configDirFile.getAbsolutePath());
+ if(propertiesDirFile.exists() && propertiesDirFile.isDirectory()) {
+ configDirectory = new ConfigurationDirectory(configDirFile.getAbsolutePath(), propertiesDirFile.getAbsolutePath());
+ } else {
+ configDirectory = new ConfigurationDirectory(configDirFile.getAbsolutePath(), null);
+ }
}
return configDirectory;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
index 7d934bb..5c8556f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
@@ -499,7 +499,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
private void populateConfigurationModules() {
//todo: can't exclude types in stack config
ConfigurationDirectory configDirectory = stackDirectory.getConfigurationDirectory(
- AmbariMetaInfo.SERVICE_CONFIG_FOLDER_NAME);
+ AmbariMetaInfo.SERVICE_CONFIG_FOLDER_NAME, AmbariMetaInfo.SERVICE_PROPERTIES_FOLDER_NAME);
if (configDirectory != null) {
for (ConfigurationModule config : configDirectory.getConfigurationModules()) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java
index e7c9c27..34b3ba1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java
@@ -246,6 +246,7 @@ public class PropertyInfo {
TEXT,
ADDITIONAL_USER_PROPERTY,
DONT_ADD_ON_UPGRADE,
- NOT_MANAGED_HDFS_PATH
+ NOT_MANAGED_HDFS_PATH,
+ VALUE_FROM_PROPERTY_FILE
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/java/org/apache/ambari/server/state/ValueAttributesInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ValueAttributesInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ValueAttributesInfo.java
index 3f7f756..931d58b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ValueAttributesInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ValueAttributesInfo.java
@@ -74,6 +74,14 @@ public class ValueAttributesInfo {
@JsonProperty("selection_cardinality")
private String selectionCardinality;
+ @XmlElement(name = "property-file-name")
+ @JsonProperty("property-file-name")
+ private String propertyFileName;
+
+ @XmlElement(name = "property-file-type")
+ @JsonProperty("property-file-type")
+ private String propertyFileType;
+
public ValueAttributesInfo() {
}
@@ -134,6 +142,22 @@ public class ValueAttributesInfo {
this.selectionCardinality = selectionCardinality;
}
+ public String getPropertyFileName() {
+ return propertyFileName;
+ }
+
+ public void setPropertyFileName(String propertyFileName) {
+ this.propertyFileName = propertyFileName;
+ }
+
+ public String getPropertyFileType() {
+ return propertyFileType;
+ }
+
+ public void setPropertyFileType(String propertyFileType) {
+ this.propertyFileType = propertyFileType;
+ }
+
public String getIncrementStep() {
return incrementStep;
}
@@ -234,6 +258,10 @@ public class ValueAttributesInfo {
if (minimum != null ? !minimum.equals(that.minimum) : that.minimum != null) return false;
if (selectionCardinality != null ? !selectionCardinality.equals(that.selectionCardinality) : that.selectionCardinality != null)
return false;
+ if (propertyFileName != null ? !propertyFileName.equals(that.propertyFileName) : that.propertyFileName != null)
+ return false;
+ if (propertyFileType != null ? !propertyFileType.equals(that.propertyFileType) : that.propertyFileType != null)
+ return false;
if (type != null ? !type.equals(that.type) : that.type != null) return false;
if (unit != null ? !unit.equals(that.unit) : that.unit != null) return false;
if (delete != null ? !delete.equals(that.delete) : that.delete != null) return false;
@@ -252,6 +280,8 @@ public class ValueAttributesInfo {
result = 31 * result + (entries != null ? entries.hashCode() : 0);
result = 31 * result + (entriesEditable != null ? entriesEditable.hashCode() : 0);
result = 31 * result + (selectionCardinality != null ? selectionCardinality.hashCode() : 0);
+ result = 31 * result + (propertyFileName != null ? propertyFileName.hashCode() : 0);
+ result = 31 * result + (propertyFileType != null ? propertyFileType.hashCode() : 0);
result = 31 * result + (incrementStep != null ? incrementStep.hashCode() : 0);
result = 31 * result + (emptyValueValid != null ? emptyValueValid.hashCode() : 0);
result = 31 * result + (visible != null ? visible.hashCode() : 0);
@@ -282,6 +312,8 @@ public class ValueAttributesInfo {
", incrementStep='" + incrementStep + '\'' +
", entriesEditable=" + entriesEditable +
", selectionCardinality='" + selectionCardinality + '\'' +
+ ", propertyFileName='" + propertyFileName + '\'' +
+ ", propertyFileType='" + propertyFileType + '\'' +
'}';
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/java/org/apache/ambari/server/utils/JsonUtils.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/JsonUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/JsonUtils.java
new file mode 100644
index 0000000..63c88d8
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/JsonUtils.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.utils;
+
+import com.google.gson.JsonParser;
+import com.google.gson.JsonSyntaxException;
+import org.eclipse.jetty.util.StringUtil;
+
+/**
+ * Static Helper methods for Json processing.
+ */
+public class JsonUtils {
+
+ public static JsonParser jsonParser = new JsonParser();
+
+ /**
+ * Checks if an input string is in valid JSON format
+ * @param jsonString input json string to validate
+ * @return <tt>true</tt> if the input string is in valid JSON format
+ */
+ public static boolean isValidJson(String jsonString) {
+
+ if(StringUtil.isBlank(jsonString)) {
+ return false;
+ }
+ try {
+ jsonParser.parse(jsonString);
+ return true;
+ } catch (JsonSyntaxException jse) {
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_script.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_script.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_script.py
index 12ca388..56051e2 100644
--- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_script.py
+++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/accumulo_script.py
@@ -36,7 +36,7 @@ from accumulo_service import accumulo_service
class AccumuloScript(Script):
# a mapping between the component named used by these scripts and the name
- # which is used by hdp-select
+ # which is used by <stack-selector-tool>
COMPONENT_TO_STACK_SELECT_MAPPING = {
"gc" : "accumulo-gc",
"master" : "accumulo-master",
@@ -51,9 +51,9 @@ class AccumuloScript(Script):
def get_stack_to_component(self):
"""
- Gets the hdp-select component name given the script component
+ Gets the <stack-selector-tool> component name given the script component
:return: the name of the component on the HDP stack which is used by
- hdp-select
+ <stack-selector-tool>
"""
if self.component not in self.COMPONENT_TO_STACK_SELECT_MAPPING:
return None
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/status_params.py
index 45dbb24..10815b0 100644
--- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/status_params.py
@@ -23,7 +23,7 @@ from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
# a map of the Ambari role to the component name
-# for use with /usr/hdp/current/<component>
+# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'ACCUMULO_MASTER' : 'accumulo-master',
'ACCUMULO_MONITOR' : 'accumulo-monitor',
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
index d000846..8fabaac 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/atlas_client.py
@@ -31,7 +31,7 @@ class AtlasClient(Script):
def get_stack_to_component(self):
return {"HDP": "atlas-client"}
- # ToDo: currently hdp-select doesn't contain atlas-client, uncomment this block when
+ # ToDo: currently <stack-selector-tool> doesn't contain atlas-client, uncomment this block when
# ToDo: atlas-client will be available
# def pre_upgrade_restart(self, env, upgrade_type=None):
# import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py
index 2c06c40..712a141 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py
@@ -25,7 +25,7 @@ from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
# a map of the Ambari role to the component name
-# for use with /usr/hdp/current/<component>
+# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'FALCON_SERVER' : 'falcon-server',
'FALCON_CLIENT' : 'falcon-client'
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
index 535c821..3ec84ee 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
@@ -25,7 +25,7 @@ from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
# a map of the Ambari role to the component name
-# for use with /usr/hdp/current/<component>
+# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'HBASE_MASTER' : 'hbase-master',
'HBASE_REGIONSERVER' : 'hbase-regionserver',
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py
index 85e7012..941b70c 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hcat_client.py
@@ -51,7 +51,7 @@ class HCatClientWindows(HCatClient):
@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
class HCatClientDefault(HCatClient):
def get_stack_to_component(self):
- # HCat client doesn't have a first-class entry in hdp-select. Since clients always
+ # HCat client doesn't have a first-class entry in <stack-selector-tool>. Since clients always
# update after daemons, this ensures that the hcat directories are correct on hosts
# which do not include the WebHCat daemon
return {"HDP": "hive-webhcat"}
@@ -59,7 +59,7 @@ class HCatClientDefault(HCatClient):
def pre_upgrade_restart(self, env, upgrade_type=None):
"""
- Execute hdp-select before reconfiguring this client to the new HDP version.
+ Execute <stack-selector-tool> before reconfiguring this client to the new HDP version.
:param env:
:param upgrade_type:
@@ -75,7 +75,7 @@ class HCatClientDefault(HCatClient):
if not params.version or compare_versions(params.version, "2.2", format=True) < 0:
return
- # HCat client doesn't have a first-class entry in hdp-select. Since clients always
+ # HCat client doesn't have a first-class entry in <stack-selector-tool>. Since clients always
# update after daemons, this ensures that the hcat directories are correct on hosts
# which do not include the WebHCat daemon
stack_select.select("hive-webhcat", params.version)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py
index c3d15e5..8045528 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_upgrade.py
@@ -53,7 +53,7 @@ def post_upgrade_deregister():
if current_hiveserver_version is None:
raise Fail('Unable to determine the current HiveServer2 version to deregister.')
- # fallback when upgrading because /usr/hdp/current/hive-server2/conf/conf.server may not exist
+ # fallback when upgrading because <stack-root>/current/hive-server2/conf/conf.server may not exist
hive_server_conf_dir = params.hive_server_conf_dir
if not os.path.exists(hive_server_conf_dir):
hive_server_conf_dir = "/etc/hive/conf.server"
@@ -62,7 +62,7 @@ def post_upgrade_deregister():
hive_execute_path = params.execute_path
# If upgrading, the upgrade-target hive binary should be used to call the --deregister command.
# If downgrading, the downgrade-source hive binary should be used to call the --deregister command.
- # By now hdp-select has been called to set 'current' to target-stack
+ # By now <stack-selector-tool> has been called to set 'current' to target-stack
if "downgrade" == params.upgrade_direction:
# hive_bin
downgrade_version = params.current_version
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 215e012..b2d0041 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -57,7 +57,7 @@ stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
stack_version_formatted_major = format_stack_version(stack_version_unformatted)
stack_is_hdp21 = Script.is_stack_less_than("2.2")
-# this is not available on INSTALL action because hdp-select is not available
+# this is not available on INSTALL action because <stack-selector-tool> is not available
stack_version_formatted = functions.get_stack_version('hive-server2')
# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade.
@@ -221,8 +221,8 @@ driver_curl_source = format("{jdk_location}/{jdbc_jar_name}")
if Script.is_stack_less_than("2.2"):
source_jdbc_file = target
else:
- # normally, the JDBC driver would be referenced by /usr/hdp/current/.../foo.jar
- # but in RU if hdp-select is called and the restart fails, then this means that current pointer
+ # normally, the JDBC driver would be referenced by <stack-root>/current/.../foo.jar
+ # but in RU if <stack-selector-tool> is called and the restart fails, then this means that current pointer
# is now pointing to the upgraded version location; that's bad for the cp command
source_jdbc_file = format("/usr/hdp/{current_version}/hive/lib/{jdbc_jar_name}")
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
index 7aeac24..62fc2e3 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
@@ -28,7 +28,7 @@ from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
# a map of the Ambari role to the component name
-# for use with /usr/hdp/current/<component>
+# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'HIVE_METASTORE' : 'hive-metastore',
'HIVE_SERVER' : 'hive-server2',
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
index 30b9a41..af0442c 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
@@ -121,7 +121,7 @@ class KnoxGatewayDefault(KnoxGateway):
# This will backup the contents of the conf directory into /tmp/knox-upgrade-backup/knox-conf-backup.tar
absolute_backup_dir = upgrade.backup_data()
- # conf-select will change the symlink to the conf folder.
+ # <conf-selector-tool> will change the symlink to the conf folder.
conf_select.select(params.stack_name, "knox", params.version)
stack_select.select("knox-server", params.version)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index 81a227e..c503a38 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -200,7 +200,7 @@ def oozie_ownership():
def prepare_war():
"""
Attempt to call prepare-war command if the marker file doesn't exist or its content doesn't equal the expected command.
- The marker file is stored in /usr/hdp/current/oozie-server/.prepare_war_cmd
+ The marker file is stored in <stack-root>/current/oozie-server/.prepare_war_cmd
"""
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
index 030fb2d..e9da71b 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
@@ -66,13 +66,13 @@ class OozieServer(Script):
if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None:
Logger.info(format("Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}"))
if compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
- # In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of
+ # In order for the "<stack-root>/current/oozie-<client/server>" point to the new version of
# oozie, we need to create the symlinks both for server and client.
# This is required as both need to be pointing to new installed oozie version.
- # Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie
+ # Sets the symlink : eg: <stack-root>/current/oozie-client -> <stack-root>/2.3.x.y-<version>/oozie
stack_select.select("oozie-client", params.version)
- # Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie
+ # Sets the symlink : eg: <stack-root>/current/oozie-server -> <stack-root>/2.3.x.y-<version>/oozie
stack_select.select("oozie-server", params.version)
if compare_versions(format_stack_version(params.version), '2.3.0.0') >= 0:
@@ -176,7 +176,7 @@ class OozieServerDefault(OozieServer):
"""
Performs the tasks that should be done before an upgrade of oozie. This includes:
- backing up configurations
- - running hdp-select and conf-select
+ - running <stack-selector-tool> and <conf-selector-tool>
- restoring configurations
- preparing the libext directory
:param env:
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
index 27e2766..d26b89d 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
@@ -99,7 +99,7 @@ class OozieUpgrade(Script):
def prepare_libext_directory():
"""
Performs the following actions on libext:
- - creates /usr/hdp/current/oozie/libext and recursively
+ - creates <stack-root>/current/oozie/libext and recursively
- set 777 permissions on it and its parents.
- downloads JDBC driver JAR if needed
- copies Falcon JAR for the Oozie WAR if needed
@@ -114,8 +114,8 @@ class OozieUpgrade(Script):
Directory(params.oozie_libext_dir, mode = 0777)
# get all hadooplzo* JAR files
- # hdp-select set hadoop-client has not run yet, therefore we cannot use
- # /usr/hdp/current/hadoop-client ; we must use params.version directly
+ # <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use
+ # <stack-root>/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
@@ -145,7 +145,7 @@ class OozieUpgrade(Script):
# copy ext ZIP to libext dir
oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
- # something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip
+ # something like <stack-root>/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip")
if not os.path.isfile(oozie_ext_zip_file):
@@ -236,7 +236,7 @@ class OozieUpgrade(Script):
Logger.info(format('Upgrading the Oozie database, using version {stack_version}'))
# the database upgrade requires the db driver JAR, but since we have
- # not yet run hdp-select to upgrade the current points, we have to use
+ # not yet run <stack-selector-tool> to upgrade the current points, we have to use
# the versioned libext directory as the location[[-vufdtffr,
versioned_libext_dir = "/usr/hdp/{0}/oozie/libext".format(stack_version)
oozie.download_database_library_if_needed(target_directory=versioned_libext_dir)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/status_params.py
index 954bb80..a08ae3a 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/status_params.py
@@ -25,7 +25,7 @@ from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
# a map of the Ambari role to the component name
-# for use with /usr/hdp/current/<component>
+# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'OOZIE_SERVER' : 'oozie-server',
'OOZIE_CLIENT' : 'oozie-client',
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py
index e5b54cd..507c00e 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py
@@ -27,7 +27,7 @@ from resource_management.libraries.functions.version import compare_versions
from resource_management.libraries.functions.constants import Direction
# a map of the Ambari role to the component name
-# for use with /usr/hdp/current/<component>
+# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'RANGER_ADMIN' : 'ranger-admin',
'RANGER_USERSYNC' : 'ranger-usersync'
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
index 906b198..3ddf356 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
@@ -36,7 +36,7 @@ from resource_management.libraries.functions.get_not_managed_resources import ge
from resource_management.libraries.script.script import Script
# a map of the Ambari role to the component name
-# for use with /usr/hdp/current/<component>
+# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'SPARK_JOBHISTORYSERVER' : 'spark-historyserver',
'SPARK_CLIENT' : 'spark-client',
@@ -57,8 +57,8 @@ host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
version = default("/commandParams/version", None)
# TODO! FIXME! Version check is not working as of today :
-# $ yum list installed | grep hdp-select
-# hdp-select.noarch 2.2.1.0-2340.el6 @HDP-2.2
+# $ yum list installed | grep <stack-selector-tool>
+# <stack-selector-tool>.noarch 2.2.1.0-2340.el6 @HDP-2.2
# And stack_version_formatted returned from hostLevelParams/stack_version is : 2.2.0.0
# Commenting out for time being
#stack_is_hdp22_or_further = stack_version_formatted != "" and compare_versions(stack_version_formatted, '2.2.1.0') >= 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
index e110c0c..b2b0816 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
@@ -24,7 +24,7 @@ from resource_management.libraries.script import Script
import os
# a map of the Ambari role to the component name
-# for use with /usr/hdp/current/<component>
+# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'SQOOP' : 'sqoop-client'
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/status_params.py
index 984a4ba..5ed0b93 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/status_params.py
@@ -23,7 +23,7 @@ from resource_management.libraries.functions import default, format
from ambari_commons import OSCheck
# a map of the Ambari role to the component name
-# for use with /usr/hdp/current/<component>
+# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'NIMBUS' : 'storm-nimbus',
'SUPERVISOR' : 'storm-supervisor',
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
index eb80ad6..0165c0b 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
@@ -53,7 +53,7 @@ tez_examples_jar = "/usr/lib/tez/tez-mapreduce-examples*.jar"
if Script.is_stack_greater_or_equal("2.2"):
tez_examples_jar = "/usr/hdp/current/tez-client/tez-examples*.jar"
-# tez only started linking /usr/hdp/x.x.x.x/tez-client/conf in HDP 2.3+
+# tez only started linking <stack-root>/x.x.x.x/tez-client/conf in HDP 2.3+
if Script.is_stack_greater_or_equal("2.3"):
# !!! use realpath for now since the symlink exists but is broken and a
# broken symlink messes with the DirectoryProvider class
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py
index dd732f5..64c2a53 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py
@@ -46,7 +46,7 @@ def refresh_tez_state_dependent_params():
global tez_home_dir, tez_conf_dir, stack_version_formatted
tez_home_dir = os.environ["TEZ_HOME"]
tez_conf_dir = os.path.join(tez_home_dir, "conf")
- # this is not available on INSTALL action because hdp-select is not available
+ # this is not available on INSTALL action because <stack-selector-tool> is not available
stack_version_formatted = get_stack_version("tez")
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index db4f32e..a3dc66c 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -34,7 +34,7 @@ from resource_management.libraries import functions
import status_params
# a map of the Ambari role to the component name
-# for use with /usr/hdp/current/<component>
+# for use with <stack-root>/current/<component>
MAPR_SERVER_ROLE_DIRECTORY_MAP = {
'HISTORYSERVER' : 'hadoop-mapreduce-historyserver',
'MAPREDUCE2_CLIENT' : 'hadoop-mapreduce-client',
@@ -52,6 +52,7 @@ config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
stack_name = default("/hostLevelParams/stack_name", None)
+tarball_map = default("/configurations/cluster-env/tarball_map", None)
# This is expected to be of the form #.#.#.#
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index e05ed60..4fdfb23 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -189,7 +189,7 @@ def yarn(name = None):
# During RU, Core Masters and Slaves need hdfs-site.xml
# TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added,
- # RU should rely on all available in /usr/hdp/<version>/hadoop/conf
+ # RU should rely on all available in <stack-root>/<version>/hadoop/conf
if 'hdfs-site' in params.config['configurations']:
XmlConfig("hdfs-site.xml",
conf_dir=params.hadoop_conf_dir,
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/status_params.py
index d18e4d7..9ae8440 100644
--- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/status_params.py
@@ -24,7 +24,7 @@ from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
# a map of the Ambari role to the component name
-# for use with /usr/hdp/current/<component>
+# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'ZOOKEEPER_SERVER' : 'zookeeper-server',
'ZOOKEEPER_CLIENT' : 'zookeeper-client'
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
index e0f3752..4020ae0 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
@@ -34,10 +34,12 @@ from ambari_commons.os_check import OSCheck, OSConst
from ambari_commons.str_utils import cbool, cint
from resource_management.libraries.functions.packages_analyzer import allInstalledPackages
from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_tools
from resource_management.libraries.functions.stack_select import get_stack_versions
from resource_management.libraries.functions.version import compare_versions, format_stack_version
from resource_management.libraries.functions.repo_version_history \
import read_actual_version_from_history_file, write_actual_version_to_history_file, REPO_VERSION_HISTORY_FILE
+from resource_management.libraries.script.script import Script
from resource_management.core.resources.system import Execute
from resource_management.core.logger import Logger
@@ -171,7 +173,7 @@ class InstallPackages(Script):
def _create_config_links_if_necessary(self, stack_id, stack_version):
"""
- Sets up the required structure for /etc/<component>/conf symlinks and /usr/hdp/current
+ Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current
configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3
stack_id: stack id, ie HDP-2.3
@@ -193,11 +195,12 @@ class InstallPackages(Script):
Logger.info("Configuration symlinks are not needed for {0}, only HDP-2.3+".format(stack_version))
return
- for package_name, directories in conf_select.PACKAGE_DIRS.iteritems():
+ for package_name, directories in conf_select.get_package_dirs().iteritems():
# if already on HDP 2.3, then we should skip making conf.backup folders
if self.current_stack_version_formatted and compare_versions(self.current_stack_version_formatted, '2.3') >= 0:
+ conf_selector_name = stack_tools.get_stack_tool_name(stack_tools.CONF_SELECTOR_NAME)
Logger.info("The current cluster stack of {0} does not require backing up configurations; "
- "only conf-select versioned config directories will be created.".format(stack_version))
+ "only {1} versioned config directories will be created.".format(stack_version, conf_selector_name))
# only link configs for all known packages
conf_select.select("HDP", package_name, stack_version, ignore_errors = True)
else:
@@ -342,8 +345,9 @@ class InstallPackages(Script):
# Install packages
packages_were_checked = False
+ stack_selector_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
try:
- Package("hdp-select",
+ Package(stack_selector_package,
action="upgrade",
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py b/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py
index e69a2e4..0be9a34 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py
@@ -37,7 +37,7 @@ class RemoveBits(Script):
"""
During Express Upgrade from HDP 2.1 to any higher version (HDP 2.2 or 2.3), the HDP 2.1 bits must be uninstalled.
This is because /usr/bin/hadoop used to be a shell script in HDP 2.1, but in HDP 2.3 it is
- a symlink to /usr/hdp/current/hadoop-client/bin/hadoop
+ a symlink to <stack-root>/current/hadoop-client/bin/hadoop
so both versions cannot coexist.
"""
Logger.info("Attempting to remove bits for HDP 2.1")
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
index b8bf176..bcb08c4 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
@@ -25,6 +25,7 @@ import shutil
from ambari_commons.os_check import OSCheck
from resource_management.libraries.script import Script
from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_tools
from resource_management.libraries.functions.constants import Direction
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.version import compare_versions
@@ -34,8 +35,6 @@ from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute, Link, Directory
-STACK_SELECT = '/usr/bin/hdp-select'
-
class UpgradeSetAll(Script):
"""
This script is a part of Rolling Upgrade workflow and is used to set the
@@ -60,12 +59,13 @@ class UpgradeSetAll(Script):
real_ver = format_stack_version(version)
if stack_name == "HDP":
if compare_versions(real_ver, min_ver) >= 0:
- cmd = ('ambari-python-wrap', STACK_SELECT, 'set', 'all', version)
+ stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
+ cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version)
code, out = shell.call(cmd, sudo=True)
if compare_versions(real_ver, format_stack_version("2.3")) >= 0:
- # backup the old and symlink /etc/[component]/conf to /usr/hdp/current/[component]
- for k, v in conf_select.PACKAGE_DIRS.iteritems():
+ # backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component]
+ for k, v in conf_select.get_package_dirs().iteritems():
for dir_def in v:
link_config(dir_def['conf_dir'], dir_def['current_dir'])
@@ -112,7 +112,7 @@ class UpgradeSetAll(Script):
return
# iterate through all directory conf mappings and undo the symlinks
- for key, value in conf_select.PACKAGE_DIRS.iteritems():
+ for key, value in conf_select.get_package_dirs().iteritems():
for directory_mapping in value:
original_config_directory = directory_mapping['conf_dir']
self._unlink_config(original_config_directory)
@@ -159,7 +159,7 @@ def link_config(old_conf, link_conf):
4. Remove the old directory and create a symlink to link_conf
:old_conf: the old config directory, ie /etc/[component]/conf
- :link_conf: the new target for the config directory, ie /usr/hdp/current/[component-dir]/conf
+ :link_conf: the new target for the config directory, ie <stack-root>/current/[component-dir]/conf
"""
if os.path.islink(old_conf):
# if the link exists but is wrong, then change it
@@ -183,7 +183,7 @@ def link_config(old_conf, link_conf):
shutil.rmtree(old_conf, ignore_errors=True)
- # link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf
+ # link /etc/[component]/conf -> <stack-root>/current/[component]-client/conf
Link(old_conf, to = link_conf)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml
index 6197d87..5f77f48 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml
@@ -151,4 +151,33 @@ gpgcheck=0</value>
<empty-value-valid>true</empty-value-valid>
</value-attributes>
</property>
+
+ <property>
+ <name>tarball_map</name>
+ <value></value>
+ <description>Tarball mappings</description>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>tarball_map.json</property-file-name>
+ <property-file-type>json</property-file-type>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>stack_tools</name>
+ <value></value>
+ <description>Stack specific tools</description>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>stack_tools.json</property-file-name>
+ <property-file-type>json</property-file-type>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>stack_root</name>
+ <value>/usr/hdp</value>
+ <description>Stack root folder</description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
index 96dc104..c772ddb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
@@ -30,7 +30,7 @@ from resource_management.libraries.script import Script
def setup_stack_symlinks():
"""
- Invokes hdp-select set all against a calculated fully-qualified, "normalized" version based on a
+ Invokes <stack-selector-tool> set all against a calculated fully-qualified, "normalized" version based on a
stack version, such as "2.3". This should always be called after a component has been
installed to ensure that all HDP pointers are correct. The stack upgrade logic does not
interact with this since it's done via a custom command and will not trigger this hook.
@@ -97,5 +97,5 @@ def link_configs(struct_out_file):
Logger.info("Could not load 'version' from {0}".format(struct_out_file))
return
- for k, v in conf_select.PACKAGE_DIRS.iteritems():
+ for k, v in conf_select.get_package_dirs().iteritems():
conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
index 07faae4..1609050 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
@@ -19,7 +19,9 @@ limitations under the License.
import os
-from resource_management import *
+from resource_management.libraries.functions import stack_tools
+from resource_management.libraries.functions.version import compare_versions
+from resource_management.core.resources.packaging import Package
def install_packages():
import params
@@ -28,7 +30,8 @@ def install_packages():
packages = ['unzip', 'curl']
if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0:
- packages.append('hdp-select')
+ stack_selector_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
+ packages.append(stack_selector_package)
Package(packages,
retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_tools.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_tools.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_tools.json
new file mode 100644
index 0000000..d1aab4b
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_tools.json
@@ -0,0 +1,4 @@
+{
+ "stack_selector": ["hdp-select", "/usr/bin/hdp-select", "hdp-select"],
+ "conf_selector": ["conf-select", "/usr/bin/conf-select", "conf-select"]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/tarball_map.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/tarball_map.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/tarball_map.json
new file mode 100644
index 0000000..98b6c69
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/tarball_map.json
@@ -0,0 +1,34 @@
+{
+ "slider": [
+ "{{ stack_root }}/{{ stack_version }}/slider/lib/slider.tar.gz",
+ "/{{ stack_name }}/apps/{{stack_version}}/slider/slider.tar.gz"
+ ],
+ "tez": [
+ "{{ stack_root }}/{{ stack_version }}/tez/lib/tez.tar.gz",
+ "/{{ stack_name }}/apps/{{ stack_version }}/tez/tez.tar.gz"
+ ],
+ "hive": [
+ "{{ stack_root }}/{{ stack_version }}/hive/hive.tar.gz",
+ "/{{ stack_name }}/apps/{{ stack_version }}/hive/hive.tar.gz"
+ ],
+ "pig": [
+ "{{ stack_root }}/{{ stack_version }}/pig/pig.tar.gz",
+ "/{{ stack_name }}/apps/{{ stack_version }}/pig/pig.tar.gz"
+ ],
+ "hadoop_streaming": [
+ "{{ stack_root }}/{{ stack_version }}/hadoop-mapreduce/hadoop-streaming.jar",
+ "/{{ stack_name }}/apps/{{ stack_version }}/mapreduce/hadoop-streaming.jar"
+ ],
+ "sqoop": [
+ "{{ stack_root }}/{{ stack_version }}/sqoop/sqoop.tar.gz",
+ "/{{ stack_name }}/apps/{{ stack_version }}/sqoop/sqoop.tar.gz"
+ ],
+ "mapreduce": [
+ "{{ stack_root }}/{{ stack_version }}/hadoop/mapreduce.tar.gz",
+ "/{{ stack_name }}/apps/{{ stack_version }}/mapreduce/mapreduce.tar.gz"
+ ],
+ "spark": [
+ "{{ stack_root }}/{{ stack_version }}/spark/lib/spark-{{ stack_name }}-assembly.jar",
+ "/{{ stack_name }}/apps/{{ stack_version }}/spark/spark-{{ stack_name }}-assembly.jar"
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
index 92c1200..962758d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
@@ -21,6 +21,7 @@ package org.apache.ambari.server.stack;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.state.CommandScriptDefinition;
import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.CustomCommandDefinition;
@@ -1147,7 +1148,7 @@ public class ServiceModuleTest {
ServiceDirectory serviceDirectory = createNiceMock(ServiceDirectory.class);
- expect(serviceDirectory.getConfigurationDirectory(dir)).andReturn(configDir).anyTimes();
+ expect(serviceDirectory.getConfigurationDirectory(dir, AmbariMetaInfo.SERVICE_PROPERTIES_FOLDER_NAME)).andReturn(configDir).anyTimes();
expect(serviceDirectory.getMetricsFile(anyObject(String.class))).andReturn(new File("testMetricsFile")).anyTimes();
expect(serviceDirectory.getWidgetsDescriptorFile(anyObject(String.class))).andReturn(new File("testWidgetsFile")).anyTimes();
expect(serviceDirectory.getAlertsFile()).andReturn(new File("testAlertsFile")).anyTimes();
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/test/java/org/apache/ambari/server/utils/TestJsonUtils.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/utils/TestJsonUtils.java b/ambari-server/src/test/java/org/apache/ambari/server/utils/TestJsonUtils.java
new file mode 100644
index 0000000..2380c50
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/utils/TestJsonUtils.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.utils;
+
+import junit.framework.Assert;
+import org.junit.Test;
+
+/**
+ * Test JsonUtils
+ */
+public class TestJsonUtils {
+ /**
+ * Tests {@link org.apache.ambari.server.utils.JsonUtils}
+ * @throws Exception
+ */
+ @Test
+ public void testIsValidJson() throws Exception {
+ Assert.assertFalse(JsonUtils.isValidJson(null));
+ Assert.assertFalse(JsonUtils.isValidJson(""));
+ Assert.assertFalse(JsonUtils.isValidJson("{"));
+ Assert.assertFalse(JsonUtils.isValidJson("}"));
+ Assert.assertTrue(JsonUtils.isValidJson("{}"));
+ Assert.assertTrue(JsonUtils.isValidJson("{ \"stack\" : \"HDP\" }"));
+ Assert.assertTrue(JsonUtils.isValidJson("{\n" +
+ " \"stack_selector\": [\"hdp-select\", \"/usr/bin/hdp-select\", \"hdp-select\"],\n" +
+ " \"conf_selector\": [\"conf-select\", \"/usr/bin/conf-select\", \"conf-select\"]\n" +
+ "}"));
+ Assert.assertFalse(JsonUtils.isValidJson("{\n" +
+ " \"stack_selector\": [\"hdp-select\", \"/usr/bin/hdp-select\", \"hdp-select\"],\n" +
+ " \"conf_selector\": [\"conf-select\", \"/usr/bin/conf-select\", \"conf-select\"]\n" +
+ ""));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
index 3090f6b..d3adc5a 100644
--- a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
+++ b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
@@ -183,7 +183,7 @@ class TestRUSetAll(RMFTestCase):
# verify that os.path.islink was called for each conf
self.assertTrue(islink_mock.called)
- for key, value in conf_select.PACKAGE_DIRS.iteritems():
+ for key, value in conf_select.get_package_dirs().iteritems():
for directory_mapping in value:
original_config_directory = directory_mapping['conf_dir']
is_link_called = False
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/test/python/host_scripts/TestAlertDiskSpace.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/host_scripts/TestAlertDiskSpace.py b/ambari-server/src/test/python/host_scripts/TestAlertDiskSpace.py
index 2608050..90a5a1e 100644
--- a/ambari-server/src/test/python/host_scripts/TestAlertDiskSpace.py
+++ b/ambari-server/src/test/python/host_scripts/TestAlertDiskSpace.py
@@ -73,7 +73,7 @@ class TestAlertDiskSpace(RMFTestCase):
self.assertEqual(res, ('WARNING', [
'Capacity Used: [30.00%, 1.6 GB], Capacity Total: [5.4 GB], path=/. Total free space is less than 5.0 GB']))
- # trigger isdir(/usr/hdp) to True
+ # trigger isdir(<stack-root>) to True
isdir_mock.return_value = True
# / OK
@@ -85,7 +85,7 @@ class TestAlertDiskSpace(RMFTestCase):
self.assertEqual(res,
('OK', ['Capacity Used: [26.28%, 5.7 GB], Capacity Total: [21.7 GB], path=/usr/hdp']))
- # /usr/hdp < 5GB
+ # <stack-root> < 5GB
disk_usage_mock.return_value = alert_disk_space.DiskInfo(
total = 5418482688L, used = 1625544806L,
free = 3792937882L, path="/usr/hdp")
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
index b5b43d6..a165718 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
@@ -84,7 +84,7 @@ class Test(RMFTestCase):
self.assertResourceCalled("Execute", ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hadoop-client', '2.2.1.0-2067'), sudo=True)
- # for now, it's enough that hdp-select is confirmed
+ # for now, it's enough that <stack-selector-tool> is confirmed
@patch("resource_management.libraries.functions.security_commons.build_expectations")
@patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py
index efe6038..de7d641 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py
@@ -385,7 +385,7 @@ class TestMapReduce2Client(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- # for now, it's enough that hdp-select is confirmed
+ # for now, it's enough that <stack-selector-tool> is confirmed
self.assertResourceCalled("Execute", ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hadoop-client', '2.2.1.0-2067'), sudo=True)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py
index 4601092..9f7770c 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py
@@ -556,7 +556,7 @@ class TestYarnClient(RMFTestCase):
self.assertResourceCalled("Execute", ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hadoop-client', '2.2.1.0-2067'), sudo=True)
- # for now, it's enough that hdp-select is confirmed
+ # for now, it's enough that <stack-selector-tool> is confirmed
@patch.object(functions, "get_stack_version", new = MagicMock(return_value='2.3.0.0-1234'))
def test_pre_upgrade_restart_23(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
index 31dac56..812edc3 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
@@ -80,6 +80,14 @@ class TestHookAfterInstall(RMFTestCase):
configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
only_if="ls /usr/hdp/current/hadoop-client/conf")
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/knox/conf', '/etc/knox/conf.backup'),
+ not_if = 'test -e /etc/knox/conf.backup',
+ sudo = True)
+ self.assertResourceCalled('Directory', '/etc/knox/conf',
+ action = ['delete'])
+ self.assertResourceCalled('Link', '/etc/knox/conf',
+ to = '/usr/hdp/current/knox-server/conf')
+
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/ranger/kms/conf', '/etc/ranger/kms/conf.backup'),
not_if = 'test -e /etc/ranger/kms/conf.backup',
sudo = True,)
@@ -96,29 +104,37 @@ class TestHookAfterInstall(RMFTestCase):
self.assertResourceCalled('Link', '/etc/hive2/conf',
to = '/usr/hdp/current/hive-server2-hive2/conf')
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/zookeeper/conf', '/etc/zookeeper/conf.backup'),
- not_if = 'test -e /etc/zookeeper/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/zookeeper/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/zookeeper/conf',
- to = '/usr/hdp/current/zookeeper-client/conf')
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/sqoop/conf', '/etc/sqoop/conf.backup'),
+ not_if = 'test -e /etc/sqoop/conf.backup',
+ sudo = True)
+ self.assertResourceCalled('Directory', '/etc/sqoop/conf',
+ action = ['delete'])
+ self.assertResourceCalled('Link', '/etc/sqoop/conf',
+ to = '/usr/hdp/current/sqoop-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/pig/conf', '/etc/pig/conf.backup'),
- not_if = 'test -e /etc/pig/conf.backup',
- sudo = True,)
+ not_if = 'test -e /etc/pig/conf.backup',
+ sudo = True,)
self.assertResourceCalled('Directory', '/etc/pig/conf',
- action = ['delete'],)
+ action = ['delete'],)
self.assertResourceCalled('Link', '/etc/pig/conf',
- to = '/usr/hdp/current/pig-client/conf',)
+ to = '/usr/hdp/current/pig-client/conf',)
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/tez/conf', '/etc/tez/conf.backup'),
- not_if = 'test -e /etc/tez/conf.backup',
- sudo = True)
+ not_if = 'test -e /etc/tez/conf.backup',
+ sudo = True)
self.assertResourceCalled('Directory', '/etc/tez/conf',
- action = ['delete'])
+ action = ['delete'])
self.assertResourceCalled('Link', '/etc/tez/conf',
- to = '/usr/hdp/current/tez-client/conf')
+ to = '/usr/hdp/current/tez-client/conf')
+
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/kafka/conf', '/etc/kafka/conf.backup'),
+ not_if = 'test -e /etc/kafka/conf.backup',
+ sudo = True)
+ self.assertResourceCalled('Directory', '/etc/kafka/conf',
+ action = ['delete'])
+ self.assertResourceCalled('Link', '/etc/kafka/conf',
+ to = '/usr/hdp/current/kafka-broker/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive-webhcat/conf', '/etc/hive-webhcat/conf.backup'),
not_if = 'test -e /etc/hive-webhcat/conf.backup',
@@ -145,29 +161,14 @@ class TestHookAfterInstall(RMFTestCase):
self.assertResourceCalled('Link', '/etc/hbase/conf',
to = '/usr/hdp/current/hbase-client/conf')
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/knox/conf', '/etc/knox/conf.backup'),
- not_if = 'test -e /etc/knox/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/knox/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/knox/conf',
- to = '/usr/hdp/current/knox-server/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/ranger/usersync/conf', '/etc/ranger/usersync/conf.backup'),
- not_if = 'test -e /etc/ranger/usersync/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/ranger/usersync/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/ranger/usersync/conf',
- to = '/usr/hdp/current/ranger-usersync/conf')
-
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hadoop/conf', '/etc/hadoop/conf.backup'),
- not_if = 'test -e /etc/hadoop/conf.backup',
- sudo = True)
+ not_if = 'test -e /etc/hadoop/conf.backup',
+ sudo = True)
self.assertResourceCalled('Directory', '/etc/hadoop/conf',
- action = ['delete'])
+ action = ['delete'])
self.assertResourceCalled('Link', '/etc/hadoop/conf',
- to = '/usr/hdp/current/hadoop-client/conf')
+ to = '/usr/hdp/current/hadoop-client/conf')
+
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/mahout/conf', '/etc/mahout/conf.backup'),
not_if = 'test -e /etc/mahout/conf.backup',
@@ -201,13 +202,13 @@ class TestHookAfterInstall(RMFTestCase):
self.assertResourceCalled('Link', '/etc/flume/conf',
to = '/usr/hdp/current/flume-server/conf',)
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/sqoop/conf', '/etc/sqoop/conf.backup'),
- not_if = 'test -e /etc/sqoop/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/sqoop/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/sqoop/conf',
- to = '/usr/hdp/current/sqoop-client/conf')
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/zookeeper/conf', '/etc/zookeeper/conf.backup'),
+ not_if = 'test -e /etc/zookeeper/conf.backup',
+ sudo = True)
+ self.assertResourceCalled('Directory', '/etc/zookeeper/conf',
+ action = ['delete'])
+ self.assertResourceCalled('Link', '/etc/zookeeper/conf',
+ to = '/usr/hdp/current/zookeeper-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/accumulo/conf', '/etc/accumulo/conf.backup'),
not_if = 'test -e /etc/accumulo/conf.backup',
@@ -257,7 +258,6 @@ class TestHookAfterInstall(RMFTestCase):
self.assertResourceCalled('Link', '/etc/falcon/conf',
to = '/usr/hdp/current/falcon-client/conf')
-
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/spark/conf', '/etc/spark/conf.backup'),
not_if = 'test -e /etc/spark/conf.backup',
sudo = True)
@@ -266,13 +266,13 @@ class TestHookAfterInstall(RMFTestCase):
self.assertResourceCalled('Link', '/etc/spark/conf',
to = '/usr/hdp/current/spark-client/conf')
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/kafka/conf', '/etc/kafka/conf.backup'),
- not_if = 'test -e /etc/kafka/conf.backup',
- sudo = True)
- self.assertResourceCalled('Directory', '/etc/kafka/conf',
- action = ['delete'])
- self.assertResourceCalled('Link', '/etc/kafka/conf',
- to = '/usr/hdp/current/kafka-broker/conf')
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/ranger/usersync/conf', '/etc/ranger/usersync/conf.backup'),
+ not_if = 'test -e /etc/ranger/usersync/conf.backup',
+ sudo = True)
+ self.assertResourceCalled('Directory', '/etc/ranger/usersync/conf',
+ action = ['delete'])
+ self.assertResourceCalled('Link', '/etc/ranger/usersync/conf',
+ to = '/usr/hdp/current/ranger-usersync/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive/conf', '/etc/hive/conf.backup'),
not_if = 'test -e /etc/hive/conf.backup',
@@ -327,6 +327,13 @@ class TestHookAfterInstall(RMFTestCase):
configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
only_if="ls /usr/hdp/current/hadoop-client/conf")
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/knox/conf', '/etc/knox/conf.backup'),
+ not_if = 'test -e /etc/knox/conf.backup',
+ sudo = True)
+ self.assertResourceCalled('Directory', '/etc/knox/conf',
+ action = ['delete'])
+ self.assertResourceCalled('Link', '/etc/knox/conf',
+ to = '/usr/hdp/current/knox-server/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/ranger/kms/conf', '/etc/ranger/kms/conf.backup'),
not_if = 'test -e /etc/ranger/kms/conf.backup',
sudo = True,)
@@ -345,15 +352,13 @@ class TestHookAfterInstall(RMFTestCase):
self.assertResourceCalled('Link', '/etc/hive2/conf',
to = '/usr/hdp/current/hive-server2-hive2/conf')
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/zookeeper/conf', '/etc/zookeeper/conf.backup'),
- not_if = 'test -e /etc/zookeeper/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/zookeeper/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/zookeeper/conf',
- to = '/usr/hdp/current/zookeeper-client/conf')
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/sqoop/conf', '/etc/sqoop/conf.backup'),
+ not_if = 'test -e /etc/sqoop/conf.backup',
+ sudo = True)
+ self.assertResourceCalled('Directory', '/etc/sqoop/conf',
+ action = ['delete'])
+ self.assertResourceCalled('Link', '/etc/sqoop/conf',
+ to = '/usr/hdp/current/sqoop-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/pig/conf', '/etc/pig/conf.backup'),
not_if = 'test -e /etc/pig/conf.backup',
@@ -377,6 +382,16 @@ class TestHookAfterInstall(RMFTestCase):
self.assertResourceCalled('Link', '/etc/tez/conf',
to = '/usr/hdp/current/tez-client/conf')
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/kafka/conf', '/etc/kafka/conf.backup'),
+ not_if = 'test -e /etc/kafka/conf.backup',
+ sudo = True)
+
+ self.assertResourceCalled('Directory', '/etc/kafka/conf',
+ action = ['delete'])
+
+ self.assertResourceCalled('Link', '/etc/kafka/conf',
+ to = '/usr/hdp/current/kafka-broker/conf')
+
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive-webhcat/conf', '/etc/hive-webhcat/conf.backup'),
not_if = 'test -e /etc/hive-webhcat/conf.backup',
sudo = True,)
@@ -407,26 +422,6 @@ class TestHookAfterInstall(RMFTestCase):
self.assertResourceCalled('Link', '/etc/hbase/conf',
to = '/usr/hdp/current/hbase-client/conf')
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/knox/conf', '/etc/knox/conf.backup'),
- not_if = 'test -e /etc/knox/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/knox/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/knox/conf',
- to = '/usr/hdp/current/knox-server/conf')
-
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/ranger/usersync/conf', '/etc/ranger/usersync/conf.backup'),
- not_if = 'test -e /etc/ranger/usersync/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/ranger/usersync/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/ranger/usersync/conf',
- to = '/usr/hdp/current/ranger-usersync/conf')
-
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hadoop/conf', '/etc/hadoop/conf.backup'),
not_if = 'test -e /etc/hadoop/conf.backup',
sudo = True)
@@ -477,15 +472,13 @@ class TestHookAfterInstall(RMFTestCase):
self.assertResourceCalled('Link', '/etc/flume/conf',
to = '/usr/hdp/current/flume-server/conf',)
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/sqoop/conf', '/etc/sqoop/conf.backup'),
- not_if = 'test -e /etc/sqoop/conf.backup',
- sudo = True)
-
- self.assertResourceCalled('Directory', '/etc/sqoop/conf',
- action = ['delete'])
-
- self.assertResourceCalled('Link', '/etc/sqoop/conf',
- to = '/usr/hdp/current/sqoop-client/conf')
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/zookeeper/conf', '/etc/zookeeper/conf.backup'),
+ not_if = 'test -e /etc/zookeeper/conf.backup',
+ sudo = True)
+ self.assertResourceCalled('Directory', '/etc/zookeeper/conf',
+ action = ['delete'])
+ self.assertResourceCalled('Link', '/etc/zookeeper/conf',
+ to = '/usr/hdp/current/zookeeper-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/accumulo/conf', '/etc/accumulo/conf.backup'),
not_if = 'test -e /etc/accumulo/conf.backup',
@@ -556,15 +549,15 @@ class TestHookAfterInstall(RMFTestCase):
self.assertResourceCalled('Link', '/etc/spark/conf',
to = '/usr/hdp/current/spark-client/conf')
- self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/kafka/conf', '/etc/kafka/conf.backup'),
- not_if = 'test -e /etc/kafka/conf.backup',
- sudo = True)
+ self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/ranger/usersync/conf', '/etc/ranger/usersync/conf.backup'),
+ not_if = 'test -e /etc/ranger/usersync/conf.backup',
+ sudo = True)
- self.assertResourceCalled('Directory', '/etc/kafka/conf',
- action = ['delete'])
+ self.assertResourceCalled('Directory', '/etc/ranger/usersync/conf',
+ action = ['delete'])
- self.assertResourceCalled('Link', '/etc/kafka/conf',
- to = '/usr/hdp/current/kafka-broker/conf')
+ self.assertResourceCalled('Link', '/etc/ranger/usersync/conf',
+ to = '/usr/hdp/current/ranger-usersync/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive/conf', '/etc/hive/conf.backup'),
not_if = 'test -e /etc/hive/conf.backup',
@@ -586,7 +579,7 @@ class TestHookAfterInstall(RMFTestCase):
@patch("shutil.rmtree")
def test_hook_default_stack_select_specific_version(self, rmtree_mock, symlink_mock, conf_select_select_mock, conf_select_create_mock):
"""
- Tests that hdp-select set all on a specific version, not a 2.3* wildcard is used when
+ Tests that <stack-selector-tool> set all on a specific version, not a 2.3* wildcard is used when
installing a component when the cluster version is already set.
:param rmtree_mock:
[3/3] ambari git commit: AMBARI-15420: Refactor resource_management
library (jluniya)
Posted by jl...@apache.org.
AMBARI-15420: Refactor resource_management library (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0f3bf8aa
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0f3bf8aa
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0f3bf8aa
Branch: refs/heads/trunk
Commit: 0f3bf8aa8b126a50af778d8504c73a5c8f24b334
Parents: aa8409f
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Mon Mar 21 11:45:14 2016 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Mon Mar 21 11:45:14 2016 -0700
----------------------------------------------------------------------
.../ambari_agent/HostCheckReportFileHandler.py | 2 +-
.../python/resource_management/core/utils.py | 8 +-
.../libraries/functions/conf_select.py | 175 +++++++++--------
.../libraries/functions/copy_tarball.py | 186 ++++++++++++-------
.../libraries/functions/default.py | 2 +
.../dynamic_variable_interpretation.py | 10 +-
.../libraries/functions/get_stack_version.py | 17 +-
.../libraries/functions/install_windows_msi.py | 34 ++--
.../libraries/functions/repo_version_history.py | 4 +-
.../libraries/functions/setup_ranger_plugin.py | 8 +-
.../functions/setup_ranger_plugin_xml.py | 14 +-
.../libraries/functions/stack_select.py | 94 +++++-----
.../libraries/functions/stack_tools.py | 85 +++++++++
.../libraries/functions/version_select_util.py | 9 +-
.../libraries/script/script.py | 23 ++-
.../server/api/services/AmbariMetaInfo.java | 1 +
.../server/stack/ConfigurationDirectory.java | 44 ++++-
.../ambari/server/stack/ServiceModule.java | 2 +-
.../server/stack/StackDefinitionDirectory.java | 9 +-
.../apache/ambari/server/stack/StackModule.java | 2 +-
.../ambari/server/state/PropertyInfo.java | 3 +-
.../server/state/ValueAttributesInfo.java | 32 ++++
.../apache/ambari/server/utils/JsonUtils.java | 48 +++++
.../package/scripts/accumulo_script.py | 6 +-
.../package/scripts/status_params.py | 2 +-
.../0.1.0.2.3/package/scripts/atlas_client.py | 2 +-
.../0.5.0.2.1/package/scripts/status_params.py | 2 +-
.../0.96.0.2.0/package/scripts/status_params.py | 2 +-
.../0.12.0.2.0/package/scripts/hcat_client.py | 6 +-
.../package/scripts/hive_server_upgrade.py | 4 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 6 +-
.../0.12.0.2.0/package/scripts/status_params.py | 2 +-
.../0.5.0.2.2/package/scripts/knox_gateway.py | 2 +-
.../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 2 +-
.../4.0.0.2.0/package/scripts/oozie_server.py | 8 +-
.../package/scripts/oozie_server_upgrade.py | 10 +-
.../4.0.0.2.0/package/scripts/status_params.py | 2 +-
.../RANGER/0.4.0/package/scripts/params.py | 2 +-
.../SPARK/1.2.0.2.2/package/scripts/params.py | 6 +-
.../1.4.4.2.0/package/scripts/params_linux.py | 2 +-
.../0.9.1.2.1/package/scripts/status_params.py | 2 +-
.../0.4.0.2.1/package/scripts/params_linux.py | 2 +-
.../0.4.0.2.1/package/scripts/params_windows.py | 2 +-
.../2.1.0.2.0/package/scripts/params_linux.py | 3 +-
.../YARN/2.1.0.2.0/package/scripts/yarn.py | 2 +-
.../3.4.5.2.0/package/scripts/status_params.py | 2 +-
.../custom_actions/scripts/install_packages.py | 12 +-
.../custom_actions/scripts/remove_bits.py | 2 +-
.../custom_actions/scripts/ru_set_all.py | 16 +-
.../HDP/2.0.6/configuration/cluster-env.xml | 29 +++
.../scripts/shared_initialization.py | 4 +-
.../scripts/shared_initialization.py | 7 +-
.../HDP/2.0.6/properties/stack_tools.json | 4 +
.../HDP/2.0.6/properties/tarball_map.json | 34 ++++
.../ambari/server/stack/ServiceModuleTest.java | 3 +-
.../ambari/server/utils/TestJsonUtils.java | 48 +++++
.../python/custom_actions/test_ru_set_all.py | 2 +-
.../python/host_scripts/TestAlertDiskSpace.py | 4 +-
.../stacks/2.0.6/HDFS/test_hdfs_client.py | 2 +-
.../stacks/2.0.6/YARN/test_mapreduce2_client.py | 2 +-
.../stacks/2.0.6/YARN/test_yarn_client.py | 2 +-
.../hooks/after-INSTALL/test_after_install.py | 185 +++++++++---------
.../python/stacks/2.1/TEZ/test_tez_client.py | 4 +-
.../src/main/resources/view.log4j.properties | 18 ++
.../src/main/resources/view.log4j.properties | 18 ++
.../src/main/resources/view.log4j.properties | 17 ++
.../src/main/resources/view.log4j.properties | 18 ++
.../src/main/resources/view.log4j.properties | 18 ++
.../src/main/resources/view.log4j.properties | 18 ++
.../src/main/resources/view.log4j.properties | 18 ++
.../src/main/resources/view.log4j.properties | 18 ++
71 files changed, 992 insertions(+), 402 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
index ee7db0a..940b597d 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
@@ -89,7 +89,7 @@ class HostCheckReportFileHandler:
def _stack_list_directory(self):
"""
- Return filtered list of /usr/hdp directory allowed to be removed
+ Return filtered list of <stack-root> directory allowed to be removed
:rtype list
"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/core/utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/utils.py b/ambari-common/src/main/python/resource_management/core/utils.py
index 71d0008..c39e9b5 100644
--- a/ambari-common/src/main/python/resource_management/core/utils.py
+++ b/ambari-common/src/main/python/resource_management/core/utils.py
@@ -25,6 +25,7 @@ import sys
import cStringIO
from functools import wraps
from resource_management.core.exceptions import Fail
+from itertools import chain, repeat, islice
PASSWORDS_HIDE_STRING = "[PROTECTED]"
@@ -148,4 +149,9 @@ def lazy_property(undecorated):
return v
return decorated
-
\ No newline at end of file
+
+def pad_infinite(iterable, padding=None):
+ return chain(iterable, repeat(padding))
+
+def pad(iterable, size, padding=None):
+ return islice(pad_infinite(iterable, padding), size)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
index 770595f..d4e88e7 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
@@ -18,8 +18,9 @@ limitations under the License.
"""
-__all__ = ["select", "create", "get_hadoop_conf_dir", "get_hadoop_dir"]
+__all__ = ["select", "create", "get_hadoop_conf_dir", "get_hadoop_dir", "get_package_dirs"]
+import copy
import os
import version
import stack_select
@@ -33,166 +34,169 @@ from resource_management.core.resources.system import Directory
from resource_management.core.resources.system import Execute
from resource_management.core.resources.system import Link
from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import stack_tools
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.version import compare_versions, format_stack_version
from resource_management.core.shell import as_sudo
+STACK_ROOT_PATTERN = "{{ stack_root }}"
-PACKAGE_DIRS = {
+_PACKAGE_DIRS = {
"accumulo": [
{
"conf_dir": "/etc/accumulo/conf",
- "current_dir": "/usr/hdp/current/accumulo-client/conf"
+ "current_dir": "{0}/current/accumulo-client/conf".format(STACK_ROOT_PATTERN)
}
],
"falcon": [
{
"conf_dir": "/etc/falcon/conf",
- "current_dir": "/usr/hdp/current/falcon-client/conf"
+ "current_dir": "{0}/current/falcon-client/conf".format(STACK_ROOT_PATTERN)
}
],
"hadoop": [
{
"conf_dir": "/etc/hadoop/conf",
- "current_dir": "/usr/hdp/current/hadoop-client/conf"
+ "current_dir": "{0}/current/hadoop-client/conf".format(STACK_ROOT_PATTERN)
}
],
"hbase": [
{
"conf_dir": "/etc/hbase/conf",
- "current_dir": "/usr/hdp/current/hbase-client/conf"
+ "current_dir": "{0}/current/hbase-client/conf".format(STACK_ROOT_PATTERN)
}
],
"hive": [
{
"conf_dir": "/etc/hive/conf",
- "current_dir": "/usr/hdp/current/hive-client/conf"
+ "current_dir": "{0}/current/hive-client/conf".format(STACK_ROOT_PATTERN)
}
],
"hive2": [
{
"conf_dir": "/etc/hive2/conf",
- "current_dir": "/usr/hdp/current/hive-server2-hive2/conf"
+ "current_dir": "{0}/current/hive-server2-hive2/conf".format(STACK_ROOT_PATTERN)
}
],
"kafka": [
{
"conf_dir": "/etc/kafka/conf",
- "current_dir": "/usr/hdp/current/kafka-broker/conf"
+ "current_dir": "{0}/current/kafka-broker/conf".format(STACK_ROOT_PATTERN)
}
],
"knox": [
{
"conf_dir": "/etc/knox/conf",
- "current_dir": "/usr/hdp/current/knox-server/conf"
+ "current_dir": "{0}/current/knox-server/conf".format(STACK_ROOT_PATTERN)
}
],
"mahout": [
{
"conf_dir": "/etc/mahout/conf",
- "current_dir": "/usr/hdp/current/mahout-client/conf"
+ "current_dir": "{0}/current/mahout-client/conf".format(STACK_ROOT_PATTERN)
}
],
"oozie": [
{
"conf_dir": "/etc/oozie/conf",
- "current_dir": "/usr/hdp/current/oozie-client/conf"
+ "current_dir": "{0}/current/oozie-client/conf".format(STACK_ROOT_PATTERN)
}
],
"phoenix": [
{
"conf_dir": "/etc/phoenix/conf",
- "current_dir": "/usr/hdp/current/phoenix-client/conf"
+ "current_dir": "{0}/current/phoenix-client/conf".format(STACK_ROOT_PATTERN)
}
],
"ranger-admin": [
{
"conf_dir": "/etc/ranger/admin/conf",
- "current_dir": "/usr/hdp/current/ranger-admin/conf"
+ "current_dir": "{0}/current/ranger-admin/conf".format(STACK_ROOT_PATTERN)
}
],
"ranger-kms": [
{
"conf_dir": "/etc/ranger/kms/conf",
- "current_dir": "/usr/hdp/current/ranger-kms/conf"
+ "current_dir": "{0}/current/ranger-kms/conf".format(STACK_ROOT_PATTERN)
}
],
"ranger-usersync": [
{
"conf_dir": "/etc/ranger/usersync/conf",
- "current_dir": "/usr/hdp/current/ranger-usersync/conf"
+ "current_dir": "{0}/current/ranger-usersync/conf".format(STACK_ROOT_PATTERN)
}
],
"slider": [
{
"conf_dir": "/etc/slider/conf",
- "current_dir": "/usr/hdp/current/slider-client/conf"
+ "current_dir": "{0}/current/slider-client/conf".format(STACK_ROOT_PATTERN)
}
],
"spark": [
{
"conf_dir": "/etc/spark/conf",
- "current_dir": "/usr/hdp/current/spark-client/conf"
+ "current_dir": "{0}/current/spark-client/conf".format(STACK_ROOT_PATTERN)
}
],
"sqoop": [
{
"conf_dir": "/etc/sqoop/conf",
- "current_dir": "/usr/hdp/current/sqoop-client/conf"
+ "current_dir": "{0}/current/sqoop-client/conf".format(STACK_ROOT_PATTERN)
}
],
"storm": [
{
"conf_dir": "/etc/storm/conf",
- "current_dir": "/usr/hdp/current/storm-client/conf"
+ "current_dir": "{0}/current/storm-client/conf".format(STACK_ROOT_PATTERN)
}
],
"tez": [
{
"conf_dir": "/etc/tez/conf",
- "current_dir": "/usr/hdp/current/tez-client/conf"
+ "current_dir": "{0}/current/tez-client/conf".format(STACK_ROOT_PATTERN)
}
],
"zookeeper": [
{
"conf_dir": "/etc/zookeeper/conf",
- "current_dir": "/usr/hdp/current/zookeeper-client/conf"
+ "current_dir": "{0}/current/zookeeper-client/conf".format(STACK_ROOT_PATTERN)
}
],
"pig": [
{
"conf_dir": "/etc/pig/conf",
- "current_dir": "/usr/hdp/current/pig-client/conf"
+ "current_dir": "{0}/current/pig-client/conf".format(STACK_ROOT_PATTERN)
}
],
"flume": [
{
"conf_dir": "/etc/flume/conf",
- "current_dir": "/usr/hdp/current/flume-server/conf"
+ "current_dir": "{0}/current/flume-server/conf".format(STACK_ROOT_PATTERN)
}
],
"storm-slider-client": [
{
"conf_dir": "/etc/storm-slider-client/conf",
- "current_dir": "/usr/hdp/current/storm-slider-client/conf"
+ "current_dir": "{0}/current/storm-slider-client/conf".format(STACK_ROOT_PATTERN)
}
],
"hive-hcatalog": [
{
"conf_dir": "/etc/hive-webhcat/conf",
"prefix": "/etc/hive-webhcat",
- "current_dir": "/usr/hdp/current/hive-webhcat/etc/webhcat"
+ "current_dir": "{0}/current/hive-webhcat/etc/webhcat".format(STACK_ROOT_PATTERN)
},
{
"conf_dir": "/etc/hive-hcatalog/conf",
"prefix": "/etc/hive-hcatalog",
- "current_dir": "/usr/hdp/current/hive-webhcat/etc/hcatalog"
+ "current_dir": "{0}/current/hive-webhcat/etc/hcatalog".format(STACK_ROOT_PATTERN)
}
]
}
-def get_cmd(command, package, version):
- return ('ambari-python-wrap','/usr/bin/conf-select', command, '--package', package, '--stack-version', version, '--conf-version', '0')
+def _get_cmd(command, package, version):
+ conf_selector_path = stack_tools.get_stack_tool_path(stack_tools.CONF_SELECTOR_NAME)
+ return ('ambari-python-wrap', conf_selector_path, command, '--package', package, '--stack-version', version, '--conf-version', '0')
def _valid(stack_name, package, ver):
if stack_name != "HDP":
@@ -203,12 +207,25 @@ def _valid(stack_name, package, ver):
return True
+def get_package_dirs():
+ """
+ Get package dir mappings
+ :return:
+ """
+ stack_root = Script.get_stack_root()
+ package_dirs = copy.deepcopy(_PACKAGE_DIRS)
+ for package_name, directories in package_dirs.iteritems():
+ for dir in directories:
+ current_dir = dir['current_dir']
+ current_dir = current_dir.replace(STACK_ROOT_PATTERN, stack_root)
+ dir['current_dir'] = current_dir
+ return package_dirs
def create(stack_name, package, version, dry_run = False):
"""
Creates a config version for the specified package
:param stack_name: the name of the stack
- :param package: the name of the package, as-used by conf-select
+ :param package: the name of the package, as-used by <conf-selector-tool>
:param version: the version number to create
:param dry_run: False to create the versioned config directory, True to only return what would be created
:return List of directories created
@@ -220,11 +237,11 @@ def create(stack_name, package, version, dry_run = False):
command = "dry-run-create" if dry_run else "create-conf-dir"
- code, stdout, stderr = shell.call(get_cmd(command, package, version), logoutput=False, quiet=False, sudo=True, stderr = subprocess.PIPE)
+ code, stdout, stderr = shell.call(_get_cmd(command, package, version), logoutput=False, quiet=False, sudo=True, stderr = subprocess.PIPE)
- # conf-select can set more than one directory
+ # <conf-selector-tool> can set more than one directory
# per package, so return that list, especially for dry_run
- # > conf-select dry-run-create --package hive-hcatalog --stack-version 2.4.0.0-169 0
+ # > <conf-selector-tool> dry-run-create --package hive-hcatalog --stack-version 2.4.0.0-169 0
# /etc/hive-webhcat/2.4.0.0-169/0
# /etc/hive-hcatalog/2.4.0.0-169/0
created_directories = []
@@ -248,7 +265,7 @@ def select(stack_name, package, version, try_create=True, ignore_errors=False):
"""
Selects a config version for the specified package.
:param stack_name: the name of the stack
- :param package: the name of the package, as-used by conf-select
+ :param package: the name of the package, as-used by <conf-selector-tool>
:param version: the version number to create
:param try_create: optional argument to attempt to create the directory before setting it
:param ignore_errors: optional argument to ignore any error and simply log a warning
@@ -260,15 +277,16 @@ def select(stack_name, package, version, try_create=True, ignore_errors=False):
if try_create:
create(stack_name, package, version)
- shell.checked_call(get_cmd("set-conf-dir", package, version), logoutput=False, quiet=False, sudo=True)
+ shell.checked_call(_get_cmd("set-conf-dir", package, version), logoutput=False, quiet=False, sudo=True)
# for consistency sake, we must ensure that the /etc/<component>/conf symlink exists and
- # points to /usr/hdp/current/<component>/conf - this is because some people still prefer to
- # use /etc/<component>/conf even though /usr/hdp is the "future"
- if package in PACKAGE_DIRS:
+ # points to <stack-root>/current/<component>/conf - this is because some people still prefer to
+ # use /etc/<component>/conf even though <stack-root> is the "future"
+ package_dirs = get_package_dirs()
+ if package in package_dirs:
Logger.info("Ensuring that {0} has the correct symlink structure".format(package))
- directory_list = PACKAGE_DIRS[package]
+ directory_list = package_dirs[package]
for directory_structure in directory_list:
conf_dir = directory_structure["conf_dir"]
current_dir = directory_structure["current_dir"]
@@ -299,30 +317,31 @@ def get_hadoop_conf_dir(force_latest_on_upgrade=False):
"""
Gets the shared hadoop conf directory using:
1. Start with /etc/hadoop/conf
- 2. When the stack is greater than HDP-2.2, use /usr/hdp/current/hadoop-client/conf
+ 2. When the stack is greater than HDP-2.2, use <stack-root>/current/hadoop-client/conf
3. Only when doing a RU and HDP-2.3 or higher, use the value as computed
- by conf-select. This is in the form /usr/hdp/VERSION/hadoop/conf to make sure
+ by <conf-selector-tool>. This is in the form <stack-root>/VERSION/hadoop/conf to make sure
the configs are written in the correct place. However, if the component itself has
not yet been upgraded, it should use the hadoop configs from the prior version.
- This will perform an hdp-select status to determine which version to use.
+ This will perform an <stack-selector-tool> status to determine which version to use.
:param force_latest_on_upgrade: if True, then force the returned path to always
- be that of the upgrade target version, even if hdp-select has not been called. This
+ be that of the upgrade target version, even if <stack-selector-tool> has not been called. This
is primarily used by hooks like before-ANY to ensure that hadoop environment
configurations are written to the correct location since they are written out
- before the hdp-select/conf-select would have been called.
+ before the <stack-selector-tool>/<conf-selector-tool> would have been called.
"""
hadoop_conf_dir = "/etc/hadoop/conf"
stack_name = None
+ stack_root = Script.get_stack_root()
version = None
allow_setting_conf_select_symlink = False
if not Script.in_stack_upgrade():
# During normal operation, the HDP stack must be 2.3 or higher
if Script.is_stack_greater_or_equal("2.2"):
- hadoop_conf_dir = "/usr/hdp/current/hadoop-client/conf"
+ hadoop_conf_dir = os.path.join(stack_root, "current", "hadoop-client", "conf")
if Script.is_stack_greater_or_equal("2.3"):
- hadoop_conf_dir = "/usr/hdp/current/hadoop-client/conf"
+ hadoop_conf_dir = os.path.join(stack_root, "current", "hadoop-client", "conf")
stack_name = default("/hostLevelParams/stack_name", None)
version = default("/commandParams/version", None)
@@ -337,13 +356,13 @@ def get_hadoop_conf_dir(force_latest_on_upgrade=False):
Type__|_Source_|_Target_|_Direction_____________|_Comment_____________________________________________________________
Normal| | 2.2 | | Use /etc/hadoop/conf
- Normal| | 2.3 | | Use /etc/hadoop/conf, which should be a symlink to /usr/hdp/current/hadoop-client/conf
- EU | 2.1 | 2.3 | Upgrade | Use versioned /usr/hdp/current/hadoop-client/conf
+ Normal| | 2.3 | | Use /etc/hadoop/conf, which should be a symlink to <stack-root>/current/hadoop-client/conf
+ EU | 2.1 | 2.3 | Upgrade | Use versioned <stack-root>/current/hadoop-client/conf
| | | No Downgrade Allowed | Invalid
- EU/RU | 2.2 | 2.2.* | Any | Use /usr/hdp/current/hadoop-client/conf
- EU/RU | 2.2 | 2.3 | Upgrade | Use /usr/hdp/$version/hadoop/conf, which should be a symlink destination
- | | | Downgrade | Use /usr/hdp/current/hadoop-client/conf
- EU/RU | 2.3 | 2.3.* | Any | Use /usr/hdp/$version/hadoop/conf, which should be a symlink destination
+ EU/RU | 2.2 | 2.2.* | Any | Use <stack-root>/current/hadoop-client/conf
+ EU/RU | 2.2 | 2.3 | Upgrade | Use <stack-root>/$version/hadoop/conf, which should be a symlink destination
+ | | | Downgrade | Use <stack-root>/current/hadoop-client/conf
+ EU/RU | 2.3 | 2.3.* | Any | Use <stack-root>/$version/hadoop/conf, which should be a symlink destination
'''
# The method "is_stack_greater_or_equal" uses "stack_version" which is the desired stack, e.g., 2.2 or 2.3
@@ -351,7 +370,7 @@ def get_hadoop_conf_dir(force_latest_on_upgrade=False):
# In an RU Downgrade from HDP 2.3 to 2.2, the first thing we do is
# rm /etc/[component]/conf and then mv /etc/[component]/conf.backup /etc/[component]/conf
if Script.is_stack_greater_or_equal("2.2"):
- hadoop_conf_dir = "/usr/hdp/current/hadoop-client/conf"
+ hadoop_conf_dir = os.path.join(stack_root, "current", "hadoop-client", "conf")
# This contains the "version", including the build number, that is actually used during a stack upgrade and
# is the version upgrading/downgrading to.
@@ -366,16 +385,18 @@ def get_hadoop_conf_dir(force_latest_on_upgrade=False):
Logger.info("In the middle of a stack upgrade/downgrade for Stack {0} and destination version {1}, determining which hadoop conf dir to use.".format(stack_name, version))
# This is the version either upgrading or downgrading to.
if compare_versions(format_stack_version(version), "2.3.0.0") >= 0:
- # Determine if hdp-select has been run and if not, then use the current
+ # Determine if <stack-selector-tool> has been run and if not, then use the current
# hdp version until this component is upgraded.
if not force_latest_on_upgrade:
current_stack_version = stack_select.get_role_component_current_stack_version()
if current_stack_version is not None and version != current_stack_version:
version = current_stack_version
- Logger.info("hdp-select has not yet been called to update the symlink for this component, keep using version {0}".format(current_stack_version))
+ stack_selector_name = stack_tools.get_stack_tool_name(stack_tools.STACK_SELECTOR_NAME)
+ Logger.info("{0} has not yet been called to update the symlink for this component, "
+ "keep using version {1}".format(stack_selector_name, current_stack_version))
- # Only change the hadoop_conf_dir path, don't conf-select this older version
- hadoop_conf_dir = "/usr/hdp/{0}/hadoop/conf".format(version)
+ # Only change the hadoop_conf_dir path, don't <conf-selector-tool> this older version
+ hadoop_conf_dir = os.path.join(stack_root, version, "hadoop", "conf")
Logger.info("Hadoop conf dir: {0}".format(hadoop_conf_dir))
allow_setting_conf_select_symlink = True
@@ -385,10 +406,12 @@ def get_hadoop_conf_dir(force_latest_on_upgrade=False):
# upgrading stack to version 2.3.0.0 or higher (which may be upgrade or downgrade), then consider setting the
# symlink for /etc/hadoop/conf.
# If a host does not have any HDFS or YARN components (e.g., only ZK), then it will not contain /etc/hadoop/conf
- # Therefore, any calls to conf-select will fail.
+ # Therefore, any calls to <conf-selector-tool> will fail.
# For that reason, if the hadoop conf directory exists, then make sure it is set.
if os.path.exists(hadoop_conf_dir):
- Logger.info("The hadoop conf dir {0} exists, will call conf-select on it for version {1}".format(hadoop_conf_dir, version))
+ conf_selector_name = stack_tools.get_stack_tool_name(stack_tools.CONF_SELECTOR_NAME)
+ Logger.info("The hadoop conf dir {0} exists, will call {1} on it for version {2}".format(
+ hadoop_conf_dir, conf_selector_name, version))
select(stack_name, "hadoop", version)
Logger.info("Using hadoop conf dir: {0}".format(hadoop_conf_dir))
@@ -403,18 +426,19 @@ def convert_conf_directories_to_symlinks(package, version, dirs, skip_existing_l
- Creates a /etc/<component>/conf.backup directory
- Copies all configs from /etc/<component>/conf to conf.backup
- Removes /etc/<component>/conf
- - Creates /etc/<component>/<version>/0 via conf-select
- - /usr/hdp/current/<component>-client/conf -> /etc/<component>/<version>/0 via conf-select
+ - Creates /etc/<component>/<version>/0 via <conf-selector-tool>
+ - <stack-root>/current/<component>-client/conf -> /etc/<component>/<version>/0 via <conf-selector-tool>
- Links /etc/<component>/conf to <something> depending on function paramter
- -- /etc/<component>/conf -> /usr/hdp/current/[component]-client/conf (usually)
+ -- /etc/<component>/conf -> <stack-root>/current/[component]-client/conf (usually)
-- /etc/<component>/conf -> /etc/<component>/conf.backup (only when supporting < HDP 2.3)
:param package: the package to create symlinks for (zookeeper, falcon, etc)
- :param version: the version number to use with conf-select (2.3.0.0-1234)
- :param dirs: the directories associated with the package (from PACKAGE_DIRS)
+ :param version: the version number to use with <conf-selector-tool> (2.3.0.0-1234)
+ :param dirs: the directories associated with the package (from get_package_dirs())
:param skip_existing_links: True to not do any work if already a symlink
:param link_to: link to "current" or "backup"
"""
+ stack_name = Script.get_stack_name()
bad_dirs = []
for dir_def in dirs:
if not os.path.exists(dir_def['conf_dir']):
@@ -450,7 +474,7 @@ def convert_conf_directories_to_symlinks(package, version, dirs, skip_existing_l
# we're already in the HDP stack
# Create the versioned /etc/[component]/[version]/0 folder.
# The component must be installed on the host.
- versioned_confs = create("HDP", package, version, dry_run = True)
+ versioned_confs = create(stack_name, package, version, dry_run = True)
Logger.info("Package {0} will have new conf directories: {1}".format(package, ", ".join(versioned_confs)))
@@ -460,7 +484,7 @@ def convert_conf_directories_to_symlinks(package, version, dirs, skip_existing_l
need_dirs.append(d)
if len(need_dirs) > 0:
- create("HDP", package, version)
+ create(stack_name, package, version)
# find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory
if len(dirs) > 1:
@@ -478,9 +502,9 @@ def convert_conf_directories_to_symlinks(package, version, dirs, skip_existing_l
only_if = format("ls -d {old_conf}/*"))
- # /usr/hdp/current/[component] is already set to to the correct version, e.g., /usr/hdp/[version]/[component]
+ # <stack-root>/current/[component] is already set to to the correct version, e.g., <stack-root>/[version]/[component]
- select("HDP", package, version, ignore_errors = True)
+ select(stack_name, package, version, ignore_errors = True)
# Symlink /etc/[component]/conf to /etc/[component]/conf.backup
try:
@@ -494,7 +518,7 @@ def convert_conf_directories_to_symlinks(package, version, dirs, skip_existing_l
Directory(new_symlink, action="delete")
if link_to in ["current", "backup"]:
- # link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf
+ # link /etc/[component]/conf -> <stack-root>/current/[component]-client/conf
if link_to == "backup":
Link(new_symlink, to = backup_dir)
else:
@@ -508,7 +532,7 @@ def convert_conf_directories_to_symlinks(package, version, dirs, skip_existing_l
def _seed_new_configuration_directories(package, created_directories):
"""
Copies any files from the "current" configuration directory to the directories which were
- newly created with conf-select. This function helps ensure that files which are not tracked
+ newly created with <conf-selector-tool>. This function helps ensure that files which are not tracked
by Ambari will be available after performing a stack upgrade. Although old configurations
will be copied as well, they will be overwritten when the components are writing out their
configs after upgrade during their restart.
@@ -516,18 +540,19 @@ def _seed_new_configuration_directories(package, created_directories):
This function will catch all errors, logging them, but not raising an exception. This is to
prevent problems here from stopping and otherwise healthy upgrade.
- :param package: the conf-select package name
- :param created_directories: a list of directories that conf-select said it created
+ :param package: the <conf-selector-tool> package name
+ :param created_directories: a list of directories that <conf-selector-tool> said it created
:return: None
"""
- if package not in PACKAGE_DIRS:
+ package_dirs = get_package_dirs()
+ if package not in package_dirs:
Logger.warning("Unable to seed newly created configuration directories for {0} because it is an unknown component".format(package))
return
# seed the directories with any existing configurations
# this allows files which are not tracked by Ambari to be available after an upgrade
Logger.info("Seeding versioned configuration directories for {0}".format(package))
- expected_directories = PACKAGE_DIRS[package]
+ expected_directories = package_dirs[package]
try:
# if the expected directories don't match those created, we can't seed them
@@ -539,7 +564,7 @@ def _seed_new_configuration_directories(package, created_directories):
# short circuit for a simple 1:1 mapping
if len(expected_directories) == 1:
- # /usr/hdp/current/component/conf
+ # <stack-root>/current/component/conf
# the current directory is the source of the seeded configurations;
source_seed_directory = expected_directories[0]["current_dir"]
target_seed_directory = created_directories[0]
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index 647b8b6..f07b76f 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -24,46 +24,130 @@ import os
import uuid
import tempfile
import re
+import json
from resource_management.libraries.script.script import Script
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions.default import default
from resource_management.core import shell
from resource_management.core.logger import Logger
+from resource_management.libraries.functions import stack_tools
+STACK_NAME_PATTERN = "{{ stack_name }}"
+STACK_ROOT_PATTERN = "{{ stack_root }}"
STACK_VERSION_PATTERN = "{{ stack_version }}"
-TARBALL_MAP = {
- "HDP": {
- "slider": ("/usr/hdp/{0}/slider/lib/slider.tar.gz".format(STACK_VERSION_PATTERN),
- "/hdp/apps/{0}/slider/slider.tar.gz".format(STACK_VERSION_PATTERN)),
- "tez": ("/usr/hdp/{0}/tez/lib/tez.tar.gz".format(STACK_VERSION_PATTERN),
- "/hdp/apps/{0}/tez/tez.tar.gz".format(STACK_VERSION_PATTERN)),
+_DEFAULT_TARBALL_MAP = {
+ "slider": ("{0}/{1}/slider/lib/slider.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
+ "/{0}/apps/{1}/slider/slider.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
+ "tez": ("{0}/{1}/tez/lib/tez.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
+ "/{0}/apps/{1}/tez/tez.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
+ "hive": ("{0}/{1}/hive/hive.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
+ "/{0}/apps/{1}/hive/hive.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
+ "pig": ("{0}/{1}/pig/pig.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
+ "/{0}/apps/{1}/pig/pig.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
+ "hadoop_streaming": ("{0}/{1}/hadoop-mapreduce/hadoop-streaming.jar".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
+ "/{0}/apps/{1}/mapreduce/hadoop-streaming.jar".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
+ "sqoop": ("{0}/{1}/sqoop/sqoop.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
+ "/{0}/apps/{1}/sqoop/sqoop.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
+ "mapreduce": ("{0}/{1}/hadoop/mapreduce.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
+ "/{0}/apps/{1}/mapreduce/mapreduce.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
+ "spark": ("{0}/{1}/spark/lib/spark-{2}-assembly.jar".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN, STACK_NAME_PATTERN),
+ "/{0}/apps/{1}/spark/spark-{0}-assembly.jar".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN))
+}
- "hive": ("/usr/hdp/{0}/hive/hive.tar.gz".format(STACK_VERSION_PATTERN),
- "/hdp/apps/{0}/hive/hive.tar.gz".format(STACK_VERSION_PATTERN)),
+def _get_tarball_map():
+ """
+ Get the stack-specific tarball source and destination mappings
+ :return: tarball_map
+ """
+ tarball_map_config = default("/configurations/cluster-env/tarball_map", None)
- "pig": ("/usr/hdp/{0}/pig/pig.tar.gz".format(STACK_VERSION_PATTERN),
- "/hdp/apps/{0}/pig/pig.tar.gz".format(STACK_VERSION_PATTERN)),
+ tarball_map = _DEFAULT_TARBALL_MAP
+ if tarball_map_config:
+ tarball_map = json.loads(tarball_map_config)
- "hadoop_streaming": ("/usr/hdp/{0}/hadoop-mapreduce/hadoop-streaming.jar".format(STACK_VERSION_PATTERN),
- "/hdp/apps/{0}/mapreduce/hadoop-streaming.jar".format(STACK_VERSION_PATTERN)),
+ return tarball_map
- "sqoop": ("/usr/hdp/{0}/sqoop/sqoop.tar.gz".format(STACK_VERSION_PATTERN),
- "/hdp/apps/{0}/sqoop/sqoop.tar.gz".format(STACK_VERSION_PATTERN)),
+def _get_tarball_paths(name, use_upgrading_version_during_uprade=True, custom_source_file=None, custom_dest_file=None):
- "mapreduce": ("/usr/hdp/{0}/hadoop/mapreduce.tar.gz".format(STACK_VERSION_PATTERN),
- "/hdp/apps/{0}/mapreduce/mapreduce.tar.gz".format(STACK_VERSION_PATTERN)),
+ stack_name = Script.get_stack_name()
+ if not stack_name:
+ Logger.error("Cannot copy {0} tarball to HDFS because stack name could be be determined.".format(
+ str(name)))
+ return (False, None, None)
- "spark": ("/usr/hdp/{0}/spark/lib/spark-hdp-assembly.jar".format(STACK_VERSION_PATTERN),
- "/hdp/apps/{0}/spark/spark-hdp-assembly.jar".format(STACK_VERSION_PATTERN))
- }
-}
+ stack_version = _get_current_version(use_upgrading_version_during_uprade)
+ if not stack_version:
+ Logger.error("Cannot copy {0} tarball to HDFS because stack version could be be determined.".format(
+ str(name)))
+ return (False, None, None)
+
+ stack_root = Script.get_stack_root()
+ if not stack_root:
+ Logger.error("Cannot copy {0} tarball to HDFS because stack root could be be determined.".format(
+ str(name)))
+ return (False, None, None)
+
+ tarball_map = _get_tarball_map()
+ if not tarball_map:
+ Logger.error("Cannot copy {0} tarball to HDFS because tarball map could not be determined.".format(
+ str(name), str(stack_name)))
+
+ if name is None or name.lower() not in tarball_map:
+ Logger.error("Cannot copy tarball to HDFS because {0} is not supported in stack {1} for this operation.".format(
+ str(name), str(stack_name)))
+ return (False, None, None)
+
+ (source_file, dest_file) = tarball_map[name.lower()]
+
+ if custom_source_file is not None:
+ source_file = custom_source_file
+
+ if custom_dest_file is not None:
+ dest_file = custom_dest_file
+
+ source_file = source_file.replace(STACK_NAME_PATTERN, stack_name.lower())
+ dest_file = dest_file.replace(STACK_NAME_PATTERN, stack_name.lower())
+
+ source_file = source_file.replace(STACK_ROOT_PATTERN, stack_root.lower())
+ dest_file = dest_file.replace(STACK_ROOT_PATTERN, stack_root.lower())
+
+ source_file = source_file.replace(STACK_VERSION_PATTERN, stack_version)
+ dest_file = dest_file.replace(STACK_VERSION_PATTERN, stack_version)
+ return (True, source_file, dest_file)
+
+def _get_current_version(use_upgrading_version_during_uprade=True):
+ upgrade_direction = default("/commandParams/upgrade_direction", None)
+ is_stack_upgrade = upgrade_direction is not None
+ current_version = default("/hostLevelParams/current_version", None)
+ Logger.info("Default version is {0}".format(current_version))
+ if is_stack_upgrade:
+ if use_upgrading_version_during_uprade:
+ # This is the version going to. In the case of a downgrade, it is the lower version.
+ current_version = default("/commandParams/version", None)
+ Logger.info("Because this is a Stack Upgrade, will use version {0}".format(current_version))
+ else:
+ Logger.info("This is a Stack Upgrade, but keep the version unchanged.")
+ else:
+ if current_version is None:
+ # During normal operation, the first installation of services won't yet know about the version, so must rely
+ # on <stack-selector> to get it.
+ stack_version = _get_single_version_from_stack_select()
+ if stack_version:
+ Logger.info("Will use stack version {0}".format(stack_version))
+ current_version = stack_version
+ if current_version is None:
+ message_suffix = "during stack %s" % str(upgrade_direction) if is_stack_upgrade else ""
+ Logger.warning("Cannot copy tarball because unable to determine current version {0}.".format(message_suffix))
+ return False
+
+ return current_version
def _get_single_version_from_stack_select():
"""
- Call "hdp-select versions" and return the version string if only one version is available.
+ Call "<stack-selector> versions" and return the version string if only one version is available.
:return: Returns a version string if successful, and None otherwise.
"""
# Ubuntu returns: "stdin: is not a tty", as subprocess output, so must use a temporary file to store the output.
@@ -73,7 +157,8 @@ def _get_single_version_from_stack_select():
stack_version = None
out = None
- get_stack_versions_cmd = "/usr/bin/hdp-select versions > {0}".format(tmp_file)
+ stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
+ get_stack_versions_cmd = "{0} versions > {1}".format(stack_selector_path, tmp_file)
try:
code, stdoutdata = shell.call(get_stack_versions_cmd, logoutput=True)
with open(tmp_file, 'r+') as file:
@@ -88,7 +173,7 @@ def _get_single_version_from_stack_select():
Logger.logger.exception("Could not remove file {0}. Error: {1}".format(str(tmp_file), str(e)))
if code != 0 or out is None or out == "":
- Logger.error("Could not verify HDP version by calling '{0}'. Return Code: {1}, Output: {2}.".format(get_stack_versions_cmd, str(code), str(out)))
+ Logger.error("Could not verify stack version by calling '{0}'. Return Code: {1}, Output: {2}.".format(get_stack_versions_cmd, str(code), str(out)))
return None
matches = re.findall(r"([\d\.]+\-\d+)", out)
@@ -96,7 +181,7 @@ def _get_single_version_from_stack_select():
if matches and len(matches) == 1:
stack_version = matches[0]
elif matches and len(matches) > 1:
- Logger.error("Found multiple matches for HDP version, cannot identify the correct one from: {0}".format(", ".join(matches)))
+ Logger.error("Found multiple matches for stack version, cannot identify the correct one from: {0}".format(", ".join(matches)))
return stack_version
@@ -115,59 +200,24 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
:return: Will return True if successful, otherwise, False.
"""
import params
-
- if params.stack_name is None or params.stack_name.upper() not in TARBALL_MAP:
- Logger.error("Cannot copy {0} tarball to HDFS because stack {1} does not support this operation.".format(str(name), str(params.stack_name)))
- return False
-
- if name is None or name.lower() not in TARBALL_MAP[params.stack_name.upper()]:
- Logger.warning("Cannot copy tarball to HDFS because {0} is not supported in stack {1} for this operation.".format(str(name), str(params.stack_name)))
- return False
-
Logger.info("Called copy_to_hdfs tarball: {0}".format(name))
- (source_file, dest_file) = TARBALL_MAP[params.stack_name.upper()][name.lower()]
-
- if custom_source_file is not None:
- source_file = custom_source_file
+ (success, source_file, dest_file) = _get_tarball_paths(
+ name, use_upgrading_version_during_uprade, custom_source_file, custom_dest_file)
- if custom_dest_file is not None:
- dest_file = custom_dest_file
+ if not success:
+ return False
if host_sys_prepped:
- Logger.info("Skipping copying {0} to {1} for {2} as its a sys_prepped host.".format(str(source_file), str(dest_file), str(name)))
+ Logger.info("Skipping copying {0} to {1} for {2} as its a sys_prepped host.".format(
+ str(source_file), str(dest_file), str(name)))
return True
- upgrade_direction = default("/commandParams/upgrade_direction", None)
- is_stack_upgrade = upgrade_direction is not None
- current_version = default("/hostLevelParams/current_version", None)
- Logger.info("Default version is {0}".format(current_version))
- if is_stack_upgrade:
- if use_upgrading_version_during_uprade:
- # This is the version going to. In the case of a downgrade, it is the lower version.
- current_version = default("/commandParams/version", None)
- Logger.info("Because this is a Stack Upgrade, will use version {0}".format(current_version))
- else:
- Logger.info("This is a Stack Upgrade, but keep the version unchanged.")
- else:
- if current_version is None:
- # During normal operation, the first installation of services won't yet know about the version, so must rely
- # on hdp-select to get it.
- stack_version = _get_single_version_from_stack_select()
- if stack_version:
- Logger.info("Will use stack version {0}".format(stack_version))
- current_version = stack_version
-
- if current_version is None:
- message_suffix = "during rolling %s" % str(upgrade_direction) if is_stack_upgrade else ""
- Logger.warning("Cannot copy {0} tarball because unable to determine current version {1}.".format(name, message_suffix))
- return False
-
- source_file = source_file.replace(STACK_VERSION_PATTERN, current_version)
- dest_file = dest_file.replace(STACK_VERSION_PATTERN, current_version)
Logger.info("Source file: {0} , Dest file in HDFS: {1}".format(source_file, dest_file))
if not os.path.exists(source_file):
- Logger.warning("WARNING. Cannot copy {0} tarball because file does not exist: {1} . It is possible that this component is not installed on this host.".format(str(name), str(source_file)))
+ Logger.warning("WARNING. Cannot copy {0} tarball because file does not exist: {1} . "
+ "It is possible that this component is not installed on this host.".format(
+ str(name), str(source_file)))
return False
# Because CopyFromLocal does not guarantee synchronization, it's possible for two processes to first attempt to
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/functions/default.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/default.py b/ambari-common/src/main/python/resource_management/libraries/functions/default.py
index 23383dc..061bc52 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/default.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/default.py
@@ -29,6 +29,8 @@ def default(name, default_value):
subdicts = filter(None, name.split('/'))
curr_dict = Script.get_config()
+ if not curr_dict:
+ return default_value
for x in subdicts:
if x in curr_dict:
curr_dict = curr_dict[x]
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py b/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
index a20b03c..ca8fe19 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py
@@ -28,6 +28,7 @@ from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.format import format
from resource_management.libraries.resources.copy_from_local import CopyFromLocal
from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop
+from resource_management.libraries.functions import stack_tools
from resource_management.core.resources.system import Execute
from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
@@ -54,7 +55,7 @@ def _get_tar_source_and_dest_folder(tarball_prefix):
:return: Returns a tuple of (x, y) after verifying the properties
"""
component_tar_source_file = default("/configurations/cluster-env/%s%s" % (tarball_prefix.lower(), TAR_SOURCE_SUFFIX), None)
- # E.g., /usr/hdp/current/hadoop-client/tez-{{ stack_version_formatted }}.tar.gz
+ # E.g., <stack-root>/current/hadoop-client/tez-{{ stack_version_formatted }}.tar.gz
component_tar_destination_folder = default("/configurations/cluster-env/%s%s" % (tarball_prefix.lower(), TAR_DESTINATION_FOLDER_SUFFIX), None)
# E.g., hdfs:///hdp/apps/{{ stack_version_formatted }}/mapreduce/
@@ -173,13 +174,14 @@ def copy_tarballs_to_hdfs(tarball_prefix, stack_select_component_name, component
# Ubuntu returns: "stdin: is not a tty", as subprocess output.
tmpfile = tempfile.NamedTemporaryFile()
out = None
+ (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
with open(tmpfile.name, 'r+') as file:
- get_stack_version_cmd = '/usr/bin/hdp-select status %s > %s' % (stack_select_component_name, tmpfile.name)
+ get_stack_version_cmd = '%s status %s > %s' % (stack_selector_path, stack_select_component_name, tmpfile.name)
code, stdoutdata = shell.call(get_stack_version_cmd)
out = file.read()
pass
if code != 0 or out is None:
- Logger.warning("Could not verify HDP version by calling '%s'. Return Code: %s, Output: %s." %
+ Logger.warning("Could not verify stack version by calling '%s'. Return Code: %s, Output: %s." %
(get_stack_version_cmd, str(code), str(out)))
return 1
@@ -187,7 +189,7 @@ def copy_tarballs_to_hdfs(tarball_prefix, stack_select_component_name, component
stack_version = matches[0] if matches and len(matches) > 0 else None
if not stack_version:
- Logger.error("Could not parse HDP version from output of hdp-select: %s" % str(out))
+ Logger.error("Could not parse stack version from output of %s: %s" % (stack_selector_name, str(out)))
return 1
file_name = os.path.basename(component_tar_source_file)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/functions/get_stack_version.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_stack_version.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_stack_version.py
index f2e6567..6095a80 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/get_stack_version.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_stack_version.py
@@ -30,8 +30,8 @@ from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from resource_management.core.logger import Logger
from resource_management.core.exceptions import Fail
from resource_management.core import shell
+from resource_management.libraries.functions import stack_tools
-STACK_SELECT_BINARY = "/usr/bin/hdp-select"
@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
def get_stack_version(package_name):
@@ -64,17 +64,20 @@ def get_stack_version(package_name):
"""
@param package_name, name of the package, from which, function will try to get stack version
"""
-
- if not os.path.exists(STACK_SELECT_BINARY):
- Logger.info('Skipping get_stack_version since " + STACK_SELECT_BINARY + " is not yet available')
+
+ stack_selector_path = stack_tools.get_stack_tool_name(stack_tools.STACK_SELECTOR_NAME)
+
+ if not os.path.exists(stack_selector_path):
+ Logger.info('Skipping get_stack_version since " + stack_selector_tool + " is not yet available')
return None # lazy fail
try:
- command = 'ambari-python-wrap {STACK_SELECT_BINARY} status {package_name}'.format(STACK_SELECT_BINARY=STACK_SELECT_BINARY, package_name=package_name)
+ command = 'ambari-python-wrap {stack_selector_path} status {package_name}'.format(
+ stack_selector_path=stack_selector_path, package_name=package_name)
return_code, stack_output = shell.call(command, timeout=20)
except Exception, e:
Logger.error(str(e))
- raise Fail('Unable to execute " + STACK_SELECT_BINARY + " command to retrieve the version.')
+ raise Fail('Unable to execute " + stack_selector_path + " command to retrieve the version.')
if return_code != 0:
raise Fail(
@@ -85,7 +88,7 @@ def get_stack_version(package_name):
match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', stack_version)
if match is None:
- Logger.info('Failed to get extracted version with ' + STACK_SELECT_BINARY)
+ Logger.info('Failed to get extracted version with ' + stack_selector_path)
return None # lazy fail
return stack_version
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/functions/install_windows_msi.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/install_windows_msi.py b/ambari-common/src/main/python/resource_management/libraries/functions/install_windows_msi.py
index f1cd9cb..96d8661 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/install_windows_msi.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/install_windows_msi.py
@@ -38,24 +38,24 @@ import urlparse
__all__ = ['install_windows_msi']
msi_save_dir = None
-hdp_log_dir = "c:\\hadoop\\logs"
-hdp_data_dir = "c:\\hadoopDefaultData"
+log_dir = "c:\\hadoop\\logs"
+data_dir = "c:\\hadoopDefaultData"
local_host = socket.getfqdn()
db_flavor = "DERBY"
hdp_22 = """#Namenode Data directory
-HDFS_NAMENODE_DATA_DIR={hdp_data_dir}\\hdpdatann
+HDFS_NAMENODE_DATA_DIR={data_dir}\\hdpdatann
#Datanode Data directory
-HDFS_DATANODE_DATA_DIR={hdp_data_dir}\\hdpdatadn
+HDFS_DATANODE_DATA_DIR={data_dir}\\hdpdatadn
IS_SLIDER=yes
IS_PHOENIX=no
"""
cluster_properties = """#Log directory
-HDP_LOG_DIR={hdp_log_dir}
+HDP_LOG_DIR={log_dir}
#Data directory
-HDP_DATA_DIR={hdp_data_dir}
+HDP_DATA_DIR={data_dir}
{hdp_22_specific_props}
@@ -93,8 +93,8 @@ OOZIE_DB_USERNAME=oozie
OOZIE_DB_PASSWORD=oozie
"""
-INSTALL_MSI_CMD = 'cmd /C start /wait msiexec /qn /i {hdp_msi_path} /lv {hdp_log_path} MSIUSEREALADMINDETECTION=1 ' \
- 'HDP_LAYOUT={hdp_layout_path} DESTROY_DATA=yes HDP_USER={hadoop_user} HDP_USER_PASSWORD={hadoop_password_arg} HDP=yes ' \
+INSTALL_MSI_CMD = 'cmd /C start /wait msiexec /qn /i {msi_path} /lv {log_path} MSIUSEREALADMINDETECTION=1 ' \
+ 'HDP_LAYOUT={layout_path} DESTROY_DATA=yes HDP_USER={hadoop_user} HDP_USER_PASSWORD={hadoop_password_arg} HDP=yes ' \
'KNOX=yes KNOX_MASTER_SECRET="AmbariHDP2Windows" FALCON=yes STORM=yes HBase=yes STORM=yes FLUME=yes SLIDER=yes PHOENIX=no RANGER=no'
CREATE_SERVICE_SCRIPT = os.path.abspath("sbin\createservice.ps1")
CREATE_SERVICE_CMD = 'cmd /C powershell -ExecutionPolicy Bypass -File "{script}" -username {username} -password "{password}" -servicename ' \
@@ -176,33 +176,33 @@ def install_windows_msi(url_base, save_dir, save_files, hadoop_user, hadoop_pass
stack_version_formatted = format_stack_version(stack_version)
hdp_22_specific_props = ''
if stack_version_formatted != "" and compare_versions(stack_version_formatted, '2.2') >= 0:
- hdp_22_specific_props = hdp_22.format(hdp_data_dir=hdp_data_dir)
+ hdp_22_specific_props = hdp_22.format(data_dir=data_dir)
# MSIs cannot be larger than 2GB. HDPWIN 2.3 needed split in order to accommodate this limitation
- hdp_msi_file = ''
+ msi_file = ''
for save_file in save_files:
if save_file.lower().endswith(".msi"):
- hdp_msi_file = save_file
+ msi_file = save_file
file_url = urlparse.urljoin(url_base, save_file)
try:
download_file(file_url, os.path.join(msi_save_dir, save_file))
except:
raise Fail("Failed to download {url}".format(url=file_url))
- File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(hdp_log_dir=hdp_log_dir,
- hdp_data_dir=hdp_data_dir,
+ File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(log_dir=log_dir,
+ data_dir=data_dir,
local_host=local_host,
db_flavor=db_flavor,
hdp_22_specific_props=hdp_22_specific_props))
# install msi
- hdp_msi_path = os_utils.quote_path(os.path.join(save_dir, hdp_msi_file))
- hdp_log_path = os_utils.quote_path(os.path.join(save_dir, hdp_msi_file[:-3] + "log"))
- hdp_layout_path = os_utils.quote_path(os.path.join(save_dir, "properties.txt"))
+ msi_path = os_utils.quote_path(os.path.join(save_dir, msi_file))
+ log_path = os_utils.quote_path(os.path.join(save_dir, msi_file[:-3] + "log"))
+ layout_path = os_utils.quote_path(os.path.join(save_dir, "properties.txt"))
hadoop_password_arg = os_utils.quote_path(hadoop_password)
Execute(
- INSTALL_MSI_CMD.format(hdp_msi_path=hdp_msi_path, hdp_log_path=hdp_log_path, hdp_layout_path=hdp_layout_path,
+ INSTALL_MSI_CMD.format(msi_path=msi_path, log_path=log_path, layout_path=layout_path,
hadoop_user=hadoop_user, hadoop_password_arg=hadoop_password_arg))
reload_windows_env()
# create additional services manually due to hdp.msi limitaitons
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/functions/repo_version_history.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/repo_version_history.py b/ambari-common/src/main/python/resource_management/libraries/functions/repo_version_history.py
index d585dea..4273e9b 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/repo_version_history.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/repo_version_history.py
@@ -24,7 +24,7 @@ from resource_management.core.logger import Logger
"""
Repository version file is used while executing install_packages.py
That allows us to get actual installed version even during reinstalling existing
- repository version (when hdp-select output does not change before and after
+ repository version (when <stack-selector-tool> output does not change before and after
installing packages).
Format:
@@ -65,7 +65,7 @@ def write_actual_version_to_history_file(repository_version, actual_version):
Save the tuple of repository_version,actual_version to the repo version history file if the repository_version
doesn't already exist
:param repository_version: normalized repo version (without build number) as received from the server
- :param actual_version: Repo version with the build number, as determined using hdp-select
+ :param actual_version: Repo version with the build number, as determined using <stack-selector-tool>
:returns Return True if appended the values to the file, otherwise, return False.
"""
wrote_value = False
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py
index 4d9d8a4..3eb8591 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin.py
@@ -30,6 +30,7 @@ from resource_management.core.source import DownloadSource
from resource_management.libraries.resources import ModifyPropertiesFile
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.ranger_functions_v2 import RangeradminV2
+from resource_management.libraries.script.script import Script
def setup_ranger_plugin(component_select_name, service_name,
downloaded_custom_connector, driver_curl_source,
@@ -50,8 +51,9 @@ def setup_ranger_plugin(component_select_name, service_name,
File(driver_curl_target, mode=0644)
+ stack_root = Script.get_stack_root()
stack_version = get_stack_version(component_select_name)
- file_path = format('/usr/hdp/{stack_version}/ranger-{service_name}-plugin/install.properties')
+ file_path = format('{stack_root}/{stack_version}/ranger-{service_name}-plugin/install.properties')
if not os.path.isfile(file_path):
raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}'))
@@ -79,7 +81,9 @@ def setup_ranger_plugin(component_select_name, service_name,
else:
cmd = (format('disable-{service_name}-plugin.sh'),)
- cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{stack_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{stack_version}/ranger-{service_name}-plugin')}
+ cmd_env = {'JAVA_HOME': java_home,
+ 'PWD': format('{stack_root}/{stack_version}/ranger-{service_name}-plugin'),
+ 'PATH': format('{stack_root}/{stack_version}/ranger-{service_name}-plugin')}
Execute(cmd,
environment=cmd_env,
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
index 2ccc0c6..1046d62 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py
@@ -31,6 +31,7 @@ from resource_management.core.logger import Logger
from resource_management.core.source import DownloadSource, InlineTemplate
from resource_management.libraries.functions.ranger_functions_v2 import RangeradminV2
from resource_management.core.utils import PasswordString
+from resource_management.libraries.script.script import Script
def setup_ranger_plugin(component_select_name, service_name,
component_downloaded_custom_connector, component_driver_curl_source,
@@ -149,20 +150,21 @@ def setup_ranger_plugin(component_select_name, service_name,
def setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list):
- jar_files = os.listdir(format('/usr/hdp/{stack_version}/ranger-{service_name}-plugin/lib'))
+ stack_root = Script.get_stack_root()
+ jar_files = os.listdir(format('{stack_root}/{stack_version}/ranger-{service_name}-plugin/lib'))
for jar_file in jar_files:
for component in component_list:
- Execute(('ln','-sf',format('/usr/hdp/{stack_version}/ranger-{service_name}-plugin/lib/{jar_file}'),format('/usr/hdp/current/{component}/lib/{jar_file}')),
- not_if=format('ls /usr/hdp/current/{component}/lib/{jar_file}'),
- only_if=format('ls /usr/hdp/{stack_version}/ranger-{service_name}-plugin/lib/{jar_file}'),
+ Execute(('ln','-sf',format('{stack_root}/{stack_version}/ranger-{service_name}-plugin/lib/{jar_file}'),format('{stack_root}/current/{component}/lib/{jar_file}')),
+ not_if=format('ls {stack_root}/current/{component}/lib/{jar_file}'),
+ only_if=format('ls {stack_root}/{stack_version}/ranger-{service_name}-plugin/lib/{jar_file}'),
sudo=True)
def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_version, credential_file, xa_audit_db_password,
ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home):
- cred_lib_path = format('/usr/hdp/{stack_version}/ranger-{service_name}-plugin/install/lib/*')
- cred_setup_prefix = (format('/usr/hdp/{stack_version}/ranger-{service_name}-plugin/ranger_credential_helper.py'), '-l', cred_lib_path)
+ cred_lib_path = format('{stack_root}/{stack_version}/ranger-{service_name}-plugin/install/lib/*')
+ cred_setup_prefix = (format('{stack_root}/{stack_version}/ranger-{service_name}-plugin/ranger_credential_helper.py'), '-l', cred_lib_path)
if audit_db_is_enabled:
cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'auditDBCred', '-v', PasswordString(xa_audit_db_password), '-c', '1')
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
index 4a8eeb9..02cd7ca 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
@@ -28,17 +28,14 @@ from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.get_stack_version import get_stack_version
from resource_management.libraries.functions.format import format
from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import stack_tools
from resource_management.core.shell import call
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions.version_select_util import get_versions_from_stack_root
-STACK_SELECT = '/usr/bin/hdp-select'
-STACK_SELECT_PREFIX = ('ambari-python-wrap', STACK_SELECT)
+STACK_SELECT_PREFIX = 'ambari-python-wrap'
-# hdp-select set oozie-server 2.2.0.0-1234
-TEMPLATE = STACK_SELECT_PREFIX + ('set',)
-
-# a mapping of Ambari server role to hdp-select component name for all
+# a mapping of Ambari server role to <stack-selector-tool> component name for all
# non-clients
SERVER_ROLE_DIRECTORY_MAP = {
'ACCUMULO_MASTER' : 'accumulo-master',
@@ -75,12 +72,12 @@ SERVER_ROLE_DIRECTORY_MAP = {
'RESOURCEMANAGER' : 'hadoop-yarn-resourcemanager',
'ZOOKEEPER_SERVER' : 'zookeeper-server',
- # ZKFC is tied to NN since it doesn't have its own componnet in hdp-select and there is
+ # ZKFC is tied to NN since it doesn't have its own componnet in <stack-selector-tool> and there is
# a requirement that the ZKFC is installed on each NN
'ZKFC' : 'hadoop-hdfs-namenode'
}
-# mapping of service check to hdp-select component
+# mapping of service check to <stack-selector-tool> component
SERVICE_CHECK_DIRECTORY_MAP = {
"HDFS_SERVICE_CHECK" : "hadoop-client",
"TEZ_SERVICE_CHECK" : "hadoop-client",
@@ -90,13 +87,13 @@ SERVICE_CHECK_DIRECTORY_MAP = {
"MAHOUT_SERVICE_CHECK" : "mahout-client"
}
-# /usr/hdp/current/hadoop-client/[bin|sbin|libexec|lib]
-# /usr/hdp/2.3.0.0-1234/hadoop/[bin|sbin|libexec|lib]
-HADOOP_DIR_TEMPLATE = "/usr/hdp/{0}/{1}/{2}"
+# <stack-root>/current/hadoop-client/[bin|sbin|libexec|lib]
+# <stack-root>/2.3.0.0-1234/hadoop/[bin|sbin|libexec|lib]
+HADOOP_DIR_TEMPLATE = "{0}/{1}/{2}/{3}"
-# /usr/hdp/current/hadoop-client
-# /usr/hdp/2.3.0.0-1234/hadoop
-HADOOP_HOME_DIR_TEMPLATE = "/usr/hdp/{0}/{1}"
+# <stack-root>/current/hadoop-client
+# <stack-root>/2.3.0.0-1234/hadoop
+HADOOP_HOME_DIR_TEMPLATE = "{0}/{1}/{2}"
HADOOP_DIR_DEFAULTS = {
"home": "/usr/lib/hadoop",
@@ -108,38 +105,41 @@ HADOOP_DIR_DEFAULTS = {
def select_all(version_to_select):
"""
- Executes hdp-select on every component for the specified version. If the value passed in is a
+ Executes <stack-selector-tool> on every component for the specified version. If the value passed in is a
stack version such as "2.3", then this will find the latest installed version which
could be "2.3.0.0-9999". If a version is specified instead, such as 2.3.0.0-1234, it will use
that exact version.
- :param version_to_select: the version to hdp-select on, such as "2.3" or "2.3.0.0-1234"
+ :param version_to_select: the version to <stack-selector-tool> on, such as "2.3" or "2.3.0.0-1234"
"""
+ stack_root = Script.get_stack_root()
+ (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
# it's an error, but it shouldn't really stop anything from working
if version_to_select is None:
- Logger.error("Unable to execute hdp-select after installing because there was no version specified")
+ Logger.error(format("Unable to execute {stack_selector_name} after installing because there was no version specified"))
return
- Logger.info("Executing hdp-select set all on {0}".format(version_to_select))
+ Logger.info("Executing {0} set all on {1}".format(stack_selector_name, version_to_select))
- command = format('{sudo} /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select versions | grep ^{version_to_select} | tail -1`')
- only_if_command = format('ls -d /usr/hdp/{version_to_select}*')
+ command = format('{sudo} {stack_selector_path} set all `ambari-python-wrap {stack_selector_path} versions | grep ^{version_to_select} | tail -1`')
+ only_if_command = format('ls -d {stack_root}/{version_to_select}*')
Execute(command, only_if = only_if_command)
def select(component, version):
"""
- Executes hdp-select on the specific component and version. Some global
+ Executes <stack-selector-tool> on the specific component and version. Some global
variables that are imported via params/status_params/params_linux will need
- to be recalcuated after the hdp-select. However, python does not re-import
+ to be recalcuated after the <stack-selector-tool>. However, python does not re-import
existing modules. The only way to ensure that the configuration variables are
recalculated is to call reload(...) on each module that has global parameters.
- After invoking hdp-select, this function will also reload params, status_params,
+ After invoking <stack-selector-tool>, this function will also reload params, status_params,
and params_linux.
- :param component: the hdp-select component, such as oozie-server. If "all", then all components
+ :param component: the <stack-selector-tool> component, such as oozie-server. If "all", then all components
will be updated.
:param version: the version to set the component to, such as 2.2.0.0-1234
"""
- command = TEMPLATE + (component, version)
+ stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
+ command = (STACK_SELECT_PREFIX, stack_selector_path, "set", component, version)
Execute(command, sudo=True)
# don't trust the ordering of modules:
@@ -163,6 +163,7 @@ def get_role_component_current_stack_version():
stack_select_component = None
role = default("/role", "")
role_command = default("/roleCommand", "")
+ stack_selector_name = stack_tools.get_stack_tool_name(stack_tools.STACK_SELECTOR_NAME)
if role in SERVER_ROLE_DIRECTORY_MAP:
stack_select_component = SERVER_ROLE_DIRECTORY_MAP[role]
@@ -175,8 +176,8 @@ def get_role_component_current_stack_version():
current_stack_version = get_stack_version(stack_select_component)
if current_stack_version is None:
- Logger.warning("Unable to determine hdp-select version for {0}".format(
- stack_select_component))
+ Logger.warning("Unable to determine {0} version for {1}".format(
+ stack_selector_name, stack_select_component))
else:
Logger.info("{0} is currently at version {1}".format(
stack_select_component, current_stack_version))
@@ -188,14 +189,15 @@ def get_hadoop_dir(target, force_latest_on_upgrade=False):
"""
Return the hadoop shared directory in the following override order
1. Use default for 2.1 and lower
- 2. If 2.2 and higher, use /usr/hdp/current/hadoop-client/{target}
- 3. If 2.2 and higher AND for an upgrade, use /usr/hdp/<version>/hadoop/{target}.
- However, if the upgrade has not yet invoked hdp-select, return the current
+ 2. If 2.2 and higher, use <stack-root>/current/hadoop-client/{target}
+ 3. If 2.2 and higher AND for an upgrade, use <stack-root>/<version>/hadoop/{target}.
+ However, if the upgrade has not yet invoked <stack-selector-tool>, return the current
version of the component.
:target: the target directory
:force_latest_on_upgrade: if True, then this will return the "current" directory
- without the HDP version built into the path, such as /usr/hdp/current/hadoop-client
+ without the stack version built into the path, such as <stack-root>/current/hadoop-client
"""
+ stack_root = Script.get_stack_root()
if not target in HADOOP_DIR_DEFAULTS:
raise Fail("Target {0} not defined".format(target))
@@ -205,9 +207,9 @@ def get_hadoop_dir(target, force_latest_on_upgrade=False):
if Script.is_stack_greater_or_equal("2.2"):
# home uses a different template
if target == "home":
- hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format("current", "hadoop-client")
+ hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_root, "current", "hadoop-client")
else:
- hadoop_dir = HADOOP_DIR_TEMPLATE.format("current", "hadoop-client", target)
+ hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_root, "current", "hadoop-client", target)
# if we are not forcing "current" for HDP 2.2, then attempt to determine
# if the exact version needs to be returned in the directory
@@ -217,7 +219,7 @@ def get_hadoop_dir(target, force_latest_on_upgrade=False):
if stack_info is not None:
stack_version = stack_info[1]
- # determine if hdp-select has been run and if not, then use the current
+ # determine if <stack-selector-tool> has been run and if not, then use the current
# hdp version until this component is upgraded
current_stack_version = get_role_component_current_stack_version()
if current_stack_version is not None and stack_version != current_stack_version:
@@ -225,20 +227,21 @@ def get_hadoop_dir(target, force_latest_on_upgrade=False):
if target == "home":
# home uses a different template
- hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_version, "hadoop")
+ hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_root, stack_version, "hadoop")
else:
- hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_version, "hadoop", target)
+ hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_root, stack_version, "hadoop", target)
return hadoop_dir
def get_hadoop_dir_for_stack_version(target, stack_version):
"""
Return the hadoop shared directory for the provided stack version. This is necessary
- when folder paths of downgrade-source stack-version are needed after hdp-select.
+ when folder paths of downgrade-source stack-version are needed after <stack-selector-tool>.
:target: the target directory
:stack_version: stack version to get hadoop dir for
"""
+ stack_root = Script.get_stack_root()
if not target in HADOOP_DIR_DEFAULTS:
raise Fail("Target {0} not defined".format(target))
@@ -248,9 +251,9 @@ def get_hadoop_dir_for_stack_version(target, stack_version):
if Script.is_stack_greater_or_equal_to(formatted_stack_version, "2.2"):
# home uses a different template
if target == "home":
- hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_version, "hadoop")
+ hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_root, stack_version, "hadoop")
else:
- hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_version, "hadoop", target)
+ hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_root, stack_version, "hadoop", target)
return hadoop_dir
@@ -275,12 +278,13 @@ def _get_upgrade_stack():
def get_stack_versions(stack_root):
"""
Gets list of stack versions installed on the host.
- Be default a call to hdp-select versions is made to get the list of installed stack versions.
+ Be default a call to <stack-selector-tool> versions is made to get the list of installed stack versions.
As a fallback list of installed versions is collected from stack version directories in stack install root.
:param stack_root: Stack install root
:return: Returns list of installed stack versions.
"""
- code, out = call(STACK_SELECT_PREFIX + ('versions',))
+ stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
+ code, out = call((STACK_SELECT_PREFIX, stack_selector_path, 'versions'))
versions = []
if 0 == code:
for line in out.splitlines():
@@ -291,17 +295,19 @@ def get_stack_versions(stack_root):
def get_stack_version_before_install(component_name):
"""
- Works in the similar way to 'hdp-select status component',
+ Works in the similar way to '<stack-selector-tool> status component',
but also works for not yet installed packages.
Note: won't work if doing initial install.
"""
- component_dir = HADOOP_HOME_DIR_TEMPLATE.format("current", component_name)
+ stack_root = Script.get_stack_root()
+ component_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_root, "current", component_name)
+ stack_selector_name = stack_tools.get_stack_tool_name(stack_tools.STACK_SELECTOR_NAME)
if os.path.islink(component_dir):
stack_version = os.path.basename(os.path.dirname(os.readlink(component_dir)))
match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', stack_version)
if match is None:
- Logger.info('Failed to get extracted version with hdp-select in method get_stack_version_before_install')
+ Logger.info('Failed to get extracted version with {0} in method get_stack_version_before_install'.format(stack_selector_name))
return None # lazy fail
return stack_version
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
new file mode 100644
index 0000000..4cd7c72
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+__all__ = ["get_stack_tool", "get_stack_tool_name", "get_stack_tool_path",
+ "get_stack_tool_package", "STACK_SELECTOR_NAME", "CONF_SELECTOR_NAME"]
+
+from resource_management.core.logger import Logger
+from resource_management.core.utils import pad
+
+STACK_SELECTOR_NAME = "stack_selector"
+CONF_SELECTOR_NAME = "conf_selector"
+
+# Format
+# SELECTOR_NAME : ( "tool-name", "tool-path", "tool-package" )
+_DEFAULT_STACK_TOOLS = {
+ STACK_SELECTOR_NAME: ("hdp-select", "/usr/bin/hdp-select", "hdp-select"),
+ CONF_SELECTOR_NAME: ("conf-select", "/usr/bin/conf-select", "conf-select")
+}
+
+def get_stack_tool(name):
+ """
+ Give a tool selector name get the stack-specific tool name, tool path, tool package
+ :param name: tool selector name
+ :return: tool_name, tool_path, tool_package
+ """
+ from resource_management.libraries.functions.default import default
+ stack_tools_config = default("/configurations/cluster-env/stack_tools", None)
+ stack_tools = _DEFAULT_STACK_TOOLS
+ if stack_tools_config:
+ stack_tools = json.loads(stack_tools_config)
+
+ if name is None or name.lower() not in stack_tools:
+ Logger.warning("Cannot find config for {0} stack tool in {1}".format(str(name), str(stack_tools)))
+ return (None, None, None)
+
+ tool_config = stack_tools[name.lower()]
+
+ # Return fixed length (tool_name, tool_path tool_package) tuple
+ return tuple(pad(tool_config[:3], 3))
+
+def get_stack_tool_name(name):
+ """
+ Give a tool selector name get the stack-specific tool name
+ :param name: tool selector name
+ :return: tool_name
+ """
+ (tool_name, tool_path, tool_package) = get_stack_tool(name)
+ return tool_name
+
+
+def get_stack_tool_path(name):
+ """
+ Give a tool selector name get the stack-specific tool path
+ :param name: tool selector name
+ :return: tool_path
+ """
+ (tool_name, tool_path, tool_package) = get_stack_tool(name)
+ return tool_path
+
+
+def get_stack_tool_package(name):
+ """
+ Give a tool selector name get the stack-specific tool package
+ :param name: tool selector name
+ :return: tool_package
+ """
+ (tool_name, tool_path, tool_package) = get_stack_tool(name)
+ return tool_package
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
index 95c5cba..05e6eb6 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
@@ -25,6 +25,8 @@ import tempfile
from resource_management.core.logger import Logger
from resource_management.core import shell
+from resource_management.libraries.functions import stack_tools
+import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
def get_component_version(stack_name, component_name):
@@ -48,10 +50,11 @@ def get_component_version(stack_name, component_name):
tmpfile = tempfile.NamedTemporaryFile()
get_stack_comp_version_cmd = ""
+ (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
try:
# This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
with open(tmpfile.name, 'r') as file:
- get_stack_comp_version_cmd = '/usr/bin/hdp-select status %s > %s' % (component_name, tmpfile.name)
+ get_stack_comp_version_cmd = '%s status %s > %s' % (stack_selector_path, component_name, tmpfile.name)
code, stdoutdata = shell.call(get_stack_comp_version_cmd)
out = file.read()
@@ -62,7 +65,7 @@ def get_component_version(stack_name, component_name):
matches = re.findall(r"([\d\.]+\-\d+)", out)
version = matches[0] if matches and len(matches) > 0 else None
except Exception, e:
- Logger.error("Could not determine HDP version for component %s by calling '%s'. Return Code: %s, Output: %s." %
+ Logger.error("Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s." %
(component_name, get_stack_comp_version_cmd, str(code), str(out)))
elif stack_name == "HDPWIN":
pass
@@ -80,7 +83,7 @@ def get_component_version(stack_name, component_name):
def get_versions_from_stack_root(stack_root):
"""
- Given a stack install root (/usr/hdp), returns a list of stack versions currently installed.
+ Given a stack install root, returns a list of stack versions currently installed.
The list of installed stack versions is determined purely based on the stack version directories
found in the stack install root.
Because each stack name may have different logic, the input is a generic dictionary.
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f3bf8aa/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 1b0116e..4cbf2d7 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -43,6 +43,7 @@ from resource_management.core.resources.packaging import Package
from resource_management.libraries.functions.version_select_util import get_component_version
from resource_management.libraries.functions.version import compare_versions
from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions import stack_tools
from resource_management.libraries.functions.constants import Direction
from resource_management.libraries.functions import packages_analyzer
from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration
@@ -70,7 +71,6 @@ USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEV
"""
_PASSWORD_MAP = {"/configurations/cluster-env/hadoop.user.name":"/configurations/cluster-env/hadoop.user.password"}
-DISTRO_SELECT_PACKAGE_NAME = "hdp-select"
STACK_VERSION_PLACEHOLDER = "${stack_version}"
def get_path_from_configuration(name, configuration):
@@ -98,6 +98,7 @@ class Script(object):
3 path to service metadata dir (Directory "package" inside service directory)
4 path to file with structured command output (file will be created)
"""
+ config = None
stack_version_from_distro_select = None
structuredOut = {}
command_data_file = ""
@@ -267,9 +268,11 @@ class Script(object):
from resource_management.libraries.functions import stack_select
Script.stack_version_from_distro_select = stack_select.get_stack_version_before_install(component_name)
- # if hdp-select has not yet been done (situations like first install), we can use hdp-select version itself.
+ # If <stack-selector-tool> has not yet been done (situations like first install),
+ # we can use <stack-selector-tool> version itself.
if not Script.stack_version_from_distro_select:
- Script.stack_version_from_distro_select = packages_analyzer.getInstalledPackageVersion(DISTRO_SELECT_PACKAGE_NAME)
+ Script.stack_version_from_distro_select = packages_analyzer.getInstalledPackageVersion(
+ stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME))
return Script.stack_version_from_distro_select
@@ -323,7 +326,7 @@ class Script(object):
@staticmethod
def get_component_from_role(role_directory_map, default_role):
"""
- Gets the /usr/hdp/current/<component> component given an Ambari role,
+ Gets the <stack-root>/current/<component> component given an Ambari role,
such as DATANODE or HBASE_MASTER.
:return: the component name, such as hbase-master
"""
@@ -342,7 +345,17 @@ class Script(object):
:return: a stack name or None
"""
from resource_management.libraries.functions.default import default
- return default("/hostLevelParams/stack_name", None)
+ return default("/hostLevelParams/stack_name", "HDP")
+
+ @staticmethod
+ def get_stack_root():
+ """
+ Get the stack-specific install root directory
+ :return: stack_root
+ """
+ from resource_management.libraries.functions.default import default
+ stack_name = Script.get_stack_name()
+ return default("/configurations/cluster-env/stack_root", "/usr/{0}".format(stack_name.lower()))
@staticmethod
def get_stack_version():