You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2014/11/07 01:40:21 UTC
ambari git commit: AMBARI-8130. Provide ability to download client
configurations in Windows (Florian Barca via alejandro)
Repository: ambari
Updated Branches:
refs/heads/branch-windows-dev 2868c2051 -> 9cff20bf1
AMBARI-8130. Provide ability to download client configurations in Windows (Florian Barca via alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9cff20bf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9cff20bf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9cff20bf
Branch: refs/heads/branch-windows-dev
Commit: 9cff20bf1634a8b242a8b150c41ae5a93888adcb
Parents: 2868c20
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Thu Nov 6 16:39:31 2014 -0800
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Thu Nov 6 16:39:31 2014 -0800
----------------------------------------------------------------------
.../libraries/functions/tar_archive.py | 30 ++++++++++
.../libraries/functions/zip_archive.py | 40 +++++++++++++
.../libraries/providers/xml_config.py | 6 +-
.../libraries/script/script.py | 59 ++++++++++----------
ambari-server/conf/unix/ambari.properties | 1 +
ambari-server/conf/windows/ambari.properties | 1 +
.../src/main/assemblies/server-windows.xml | 8 +++
.../server/api/services/ComponentService.java | 6 +-
.../server/configuration/Configuration.java | 19 +++++++
.../internal/ClientConfigResourceProvider.java | 7 ++-
10 files changed, 141 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
new file mode 100644
index 0000000..efbf933
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import tarfile
+from contextlib import closing
+
+def archive_dir(output_filename, input_dir):
+ with closing(tarfile.open(output_filename, "w:gz")) as tar:
+ try:
+ tar.add(input_dir, arcname=os.path.basename("."))
+ finally:
+ tar.close()
http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-common/src/main/python/resource_management/libraries/functions/zip_archive.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/zip_archive.py b/ambari-common/src/main/python/resource_management/libraries/functions/zip_archive.py
new file mode 100644
index 0000000..cab3627
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/zip_archive.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import zipfile
+
+def _zip_dir(zip, root):
+ for dirname, dirnames, filenames in os.walk(root):
+ for filename in filenames:
+ if len(dirname) > len(root):
+ rel_path = os.path.relpath(dirname, root)
+ arch_name = rel_path + os.sep + filename
+ else:
+ arch_name = filename
+ zip.write(os.path.join(dirname, filename), arch_name)
+
+
+def archive_dir(output_filename, input_dir):
+ zipf = zipfile.ZipFile(output_filename, 'w')
+ try:
+ _zip_dir(zipf, input_dir)
+ finally:
+ zipf.close()
http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py
index 34b43e3..a7b2c21 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py
@@ -49,11 +49,11 @@ class XmlConfigProvider(Provider):
</configuration>''', extra_imports=[time], configurations_dict=self.resource.configurations,
configuration_attrs=self.resource.configuration_attributes)
-
- Logger.info(format("Generating config: {xml_config_provider_config_dir}/{filename}"))
+ xml_config_dest_file_path = os.path.join(xml_config_provider_config_dir, filename)
+ Logger.info("Generating config: {}".format(xml_config_dest_file_path))
with Environment.get_instance_copy() as env:
- File (format("{xml_config_provider_config_dir}/{filename}"),
+ File (xml_config_dest_file_path,
content = config_content,
owner = self.resource.owner,
group = self.resource.group,
http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 059dd57..d58502a 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -17,7 +17,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
-import tarfile
import tempfile
__all__ = ["Script"]
@@ -27,7 +26,6 @@ import sys
import json
import logging
import platform
-from contextlib import closing
from resource_management.libraries.resources import XmlConfig
from resource_management.core.resources import File, Directory
@@ -41,6 +39,9 @@ IS_WINDOWS = platform.system() == "Windows"
if IS_WINDOWS:
from resource_management.libraries.functions.install_hdp_msi import install_windows_msi
from resource_management.libraries.functions.reload_windows_env import reload_windows_env
+ from resource_management.libraries.functions.zip_archive import archive_dir
+else:
+ from resource_management.libraries.functions.tar_archive import archive_dir
USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEVEL> <TMP_DIR>
@@ -280,43 +281,45 @@ class Script(object):
self.fail_with_error('configure method isn\'t implemented')
def generate_configs_get_template_file_content(self, filename, dicts):
- import params
+ config = self.get_config()
content = ''
for dict in dicts.split(','):
- if dict.strip() in params.config['configurations']:
- content += params.config['configurations'][dict.strip()]['content']
+ if dict.strip() in config['configurations']:
+ content += config['configurations'][dict.strip()]['content']
return content
def generate_configs_get_xml_file_content(self, filename, dict):
- import params
- return {'configurations':params.config['configurations'][dict],
- 'configuration_attributes':params.config['configuration_attributes'][dict]}
+ config = self.get_config()
+ return {'configurations':config['configurations'][dict],
+ 'configuration_attributes':config['configuration_attributes'][dict]}
def generate_configs(self, env):
"""
Generates config files and stores them as an archive in tmp_dir
based on xml_configs_list and env_configs_list from commandParams
"""
- import params
- env.set_params(params)
- xml_configs_list = params.config['commandParams']['xml_configs_list']
- env_configs_list = params.config['commandParams']['env_configs_list']
- conf_tmp_dir = tempfile.mkdtemp()
- output_filename = os.path.join(self.get_tmp_dir(),params.config['commandParams']['output_file'])
+ config = self.get_config()
+
+ xml_configs_list = config['commandParams']['xml_configs_list']
+ env_configs_list = config['commandParams']['env_configs_list']
Directory(self.get_tmp_dir(), recursive=True)
- for file_dict in xml_configs_list:
- for filename, dict in file_dict.iteritems():
- XmlConfig(filename,
- conf_dir=conf_tmp_dir,
- **self.generate_configs_get_xml_file_content(filename, dict)
- )
- for file_dict in env_configs_list:
- for filename,dicts in file_dict.iteritems():
- File(os.path.join(conf_tmp_dir, filename),
- content=InlineTemplate(self.generate_configs_get_template_file_content(filename, dicts)))
- with closing(tarfile.open(output_filename, "w:gz")) as tar:
- tar.add(conf_tmp_dir, arcname=os.path.basename("."))
- tar.close()
- Directory(conf_tmp_dir, action="delete")
+
+ conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir())
+ output_filename = os.path.join(self.get_tmp_dir(), config['commandParams']['output_file'])
+
+ try:
+ for file_dict in xml_configs_list:
+ for filename, dict in file_dict.iteritems():
+ XmlConfig(filename,
+ conf_dir=conf_tmp_dir,
+ **self.generate_configs_get_xml_file_content(filename, dict)
+ )
+ for file_dict in env_configs_list:
+ for filename,dicts in file_dict.iteritems():
+ File(os.path.join(conf_tmp_dir, filename),
+ content=InlineTemplate(self.generate_configs_get_template_file_content(filename, dicts)))
+ archive_dir(output_filename, conf_tmp_dir)
+ finally:
+ Directory(conf_tmp_dir, action="delete")
http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/conf/unix/ambari.properties
----------------------------------------------------------------------
diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties
index 531d717..771f711 100644
--- a/ambari-server/conf/unix/ambari.properties
+++ b/ambari-server/conf/unix/ambari.properties
@@ -33,6 +33,7 @@ bootstrap.setup_agent.script=/usr/lib/python2.6/site-packages/ambari_server/setu
recommendations.dir=/var/run/ambari-server/stack-recommendations
stackadvisor.script=/var/lib/ambari-server/resources/scripts/stack_advisor.py
server.tmp.dir=/var/lib/ambari-server/tmp
+ambari.python.wrap=ambari-python-wrap
api.authenticate=true
server.connection.max.idle.millis=900000
http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/conf/windows/ambari.properties
----------------------------------------------------------------------
diff --git a/ambari-server/conf/windows/ambari.properties b/ambari-server/conf/windows/ambari.properties
index f67e7e2..fd3a7ba 100644
--- a/ambari-server/conf/windows/ambari.properties
+++ b/ambari-server/conf/windows/ambari.properties
@@ -52,6 +52,7 @@ recommendations.dir=\\var\\run\\ambari-server\\stack-recommendations
stackadvisor.script=resources\\scripts\\stack_advisor.py
server.tmp.dir=\\var\\run\\ambari-server\\tmp
views.dir=resources\\views
+ambari.python.wrap=python.exe
# Default timeout in seconds before task is killed
agent.task.timeout=600
http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/src/main/assemblies/server-windows.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/assemblies/server-windows.xml b/ambari-server/src/main/assemblies/server-windows.xml
index 84029dd..e48d3d8 100644
--- a/ambari-server/src/main/assemblies/server-windows.xml
+++ b/ambari-server/src/main/assemblies/server-windows.xml
@@ -117,6 +117,14 @@
<directory>${basedir}/../ambari-common/src/main/python/ambari_commons</directory>
<outputDirectory>ambari-server-${project.version}/sbin/ambari_commons</outputDirectory>
</fileSet>
+ <fileSet>
+ <directory>${basedir}/../ambari-common/src/main/python/ambari_jinja2/ambari_jinja2</directory>
+ <outputDirectory>ambari-server-${project.version}/sbin/ambari_jinja2</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/../ambari-common/src/main/python/resource_management</directory>
+ <outputDirectory>ambari-server-${project.version}/sbin/resource_management</outputDirectory>
+ </fileSet>
<!--
<fileSet>
<directory>${project.build.directory}/web/</directory>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java
index 5510697..e7e0029 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java
@@ -234,7 +234,7 @@ public class ComponentService extends BaseService {
Response.ResponseBuilder rb = Response.status(Response.Status.OK);
Configuration configs = new Configuration();
String tmpDir = configs.getProperty(Configuration.SERVER_TMP_DIR_KEY);
- File file = new File(tmpDir+File.separator+componentName+"-configs.tar.gz");
+ File file = new File(tmpDir + File.separator + componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION);
InputStream resultInputStream = null;
try {
resultInputStream = new FileInputStream(file);
@@ -242,8 +242,8 @@ public class ComponentService extends BaseService {
e.printStackTrace();
}
- String contentType = "application/x-ustar";
- String outputFileName = componentName + "-configs.tar.gz";
+ String contentType = Configuration.DEF_ARCHIVE_CONTENT_TYPE;
+ String outputFileName = componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION;
rb.header("Content-Disposition", "attachment; filename=\"" + outputFileName + "\"");
rb.entity(resultInputStream);
return rb.type(contentType).build();
http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index c1b5dcb..56bb5ab 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -71,6 +71,8 @@ public class Configuration {
public static final String RECOMMENDATIONS_DIR_DEFAULT = "/var/run/ambari-server/stack-recommendations";
public static final String STACK_ADVISOR_SCRIPT = "stackadvisor.script";
public static final String STACK_ADVISOR_SCRIPT_DEFAULT = "/var/lib/ambari-server/resources/scripts/stack_advisor.py";
+ public static final String AMBARI_PYTHON_WRAP_KEY = "ambari.python.wrap";
+ public static final String AMBARI_PYTHON_WRAP_DEFAULT = "python";
public static final String API_AUTHENTICATE = "api.authenticate";
public static final String API_USE_SSL = "api.ssl";
public static final String API_CSRF_PREVENTION_KEY = "api.csrfPrevention.enabled";
@@ -237,6 +239,10 @@ public class Configuration {
public static final String DEFAULT_SCHEDULER_START_DELAY_SECONDS = "120";
public static final String SERVER_TMP_DIR_KEY = "server.tmp.dir";
public static final String SERVER_TMP_DIR_DEFAULT = "/var/lib/ambari-server/tmp";
+
+ public static final String DEF_ARCHIVE_EXTENSION;
+ public static final String DEF_ARCHIVE_CONTENT_TYPE;
+
/**
* This key defines whether stages of parallel requests are executed in
* parallel or sequentally. Only stages from different requests
@@ -328,6 +334,17 @@ public class Configuration {
private volatile boolean credentialProviderInitialized = false;
private Map<String, String> customDbProperties = null;
+ static {
+ if (System.getProperty("os.name").contains("Windows")) {
+ DEF_ARCHIVE_EXTENSION = ".zip";
+ DEF_ARCHIVE_CONTENT_TYPE = "application/zip";
+ }
+ else {
+ DEF_ARCHIVE_EXTENSION = ".tar.gz";
+ DEF_ARCHIVE_CONTENT_TYPE = "application/x-ustar";
+ }
+ }
+
public Configuration() {
this(readConfigFile());
}
@@ -342,6 +359,8 @@ public class Configuration {
this.properties = properties;
configsMap = new HashMap<String, String>();
+ configsMap.put(AMBARI_PYTHON_WRAP_KEY, properties.getProperty(
+ AMBARI_PYTHON_WRAP_KEY, AMBARI_PYTHON_WRAP_DEFAULT));
configsMap.put(SRVR_TWO_WAY_SSL_KEY, properties.getProperty(
SRVR_TWO_WAY_SSL_KEY, SRVR_TWO_WAY_SSL_DEFAULT));
configsMap.put(SRVR_TWO_WAY_SSL_PORT_KEY, properties.getProperty(
http://git-wip-us.apache.org/repos/asf/ambari/blob/9cff20bf/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index 312f23f..1cfc1d0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -123,6 +123,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
Configuration configs = new Configuration();
String TMP_PATH = configs.getProperty(Configuration.SERVER_TMP_DIR_KEY);
+ String pythonCmd = configs.getProperty(Configuration.AMBARI_PYTHON_WRAP_KEY);
AmbariManagementController managementController = getManagementController();
ConfigHelper configHelper = managementController.getConfigHelper();
Cluster cluster = null;
@@ -277,7 +278,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
commandParams.put("xml_configs_list", xmlConfigs);
commandParams.put("env_configs_list", envConfigs);
- commandParams.put("output_file", componentName + "-configs.tar.gz");
+ commandParams.put("output_file", componentName + "-configs" + Configuration.DEF_ARCHIVE_EXTENSION);
Map<String, Object> jsonContent = new TreeMap<String, Object>();
jsonContent.put("configurations", configurations);
@@ -308,7 +309,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
throw new SystemException("Failed to write configurations to json file ", e);
}
- String cmd = "ambari-python-wrap " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath() + " " +
+ String cmd = pythonCmd + " " + commandScriptAbsolute + " generate_configs " + jsonFileName.getAbsolutePath() + " " +
packageFolderAbsolute + " " + TMP_PATH + File.separator + "structured-out.json" + " INFO " + TMP_PATH;
try {
@@ -331,6 +332,8 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
throw new SystemException("Controller error ", e);
}
+ Resource resource = new ResourceImpl(Resource.Type.ClientConfig);
+ resources.add(resource);
return resources;
}