You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ab...@apache.org on 2014/12/01 12:57:23 UTC
ambari git commit: AMBARI-8366 - Sqoop Configs fail to load on
Services page on WINHDP (Eugene Chekanskiy via abaranchuk)
Repository: ambari
Updated Branches:
refs/heads/trunk 5d07646ee -> f9a08f9f8
AMBARI-8366 - Sqoop Configs fail to load on Services page on WINHDP (Eugene Chekanskiy via abaranchuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f9a08f9f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f9a08f9f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f9a08f9f
Branch: refs/heads/trunk
Commit: f9a08f9f8c5100dc35c7233057f5dc5acde83ab5
Parents: 5d07646
Author: Artem Baranchuk <ab...@hortonworks.com>
Authored: Mon Dec 1 13:57:04 2014 +0200
Committer: Artem Baranchuk <ab...@hortonworks.com>
Committed: Mon Dec 1 13:57:04 2014 +0200
----------------------------------------------------------------------
.../FALCON/package/scripts/falcon_client.py | 2 +-
.../2.1/services/HDFS/package/scripts/hdfs.py | 5 +-
.../2.1/services/OOZIE/package/scripts/oozie.py | 3 +-
.../2.1/services/PIG/package/scripts/pig.py | 6 +-
.../services/SQOOP/configuration/sqoop-site.xml | 156 -------------------
.../services/SQOOP/package/scripts/params.py | 2 +-
.../2.1/services/SQOOP/package/scripts/sqoop.py | 9 +-
.../SQOOP/package/scripts/sqoop_client.py | 10 +-
.../services/STORM/package/scripts/nimbus.py | 2 +-
.../STORM/package/scripts/supervisor.py | 2 +-
.../STORM/package/scripts/yaml_config.py | 4 +-
.../services/TEZ/package/scripts/tez_client.py | 1 +
.../ZOOKEEPER/package/scripts/zookeeper.py | 6 +-
.../package/scripts/zookeeper_client.py | 1 +
14 files changed, 28 insertions(+), 181 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/FALCON/package/scripts/falcon_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/FALCON/package/scripts/falcon_client.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/FALCON/package/scripts/falcon_client.py
index a5d47da..86c7545 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/FALCON/package/scripts/falcon_client.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/FALCON/package/scripts/falcon_client.py
@@ -24,7 +24,7 @@ class FalconClient(Script):
import params
if params.falcon_home is None:
self.install_packages(env)
- self.configure(env)
+ self.configure(env)
def configure(self, env):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/HDFS/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/HDFS/package/scripts/hdfs.py
index 790c8ea..3b292f2 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/HDFS/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/HDFS/package/scripts/hdfs.py
@@ -20,6 +20,7 @@ Ambari Agent
"""
from resource_management import *
+import os
def hdfs(component=None):
import params
@@ -50,12 +51,12 @@ def hdfs(component=None):
mode="f",
configuration_attributes=params.config['configuration_attributes']['hdfs-site']
)
- File(format("{params.hadoop_conf_dir}/hadoop-metrics2.properties"),
+ File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"),
content=Template("hadoop-metrics2.properties.j2"),
owner=params.hdfs_user,
mode="f"
)
- File(format("{params.hbase_conf_dir}/hadoop-metrics2-hbase.properties"),
+ File(os.path.join(params.hbase_conf_dir, "hadoop-metrics2-hbase.properties"),
content=Template("hadoop-metrics2.properties.j2"),
owner=params.hdfs_user,
mode="f"
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/OOZIE/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/OOZIE/package/scripts/oozie.py
index e2056c4..0e59ea0 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/OOZIE/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/OOZIE/package/scripts/oozie.py
@@ -20,6 +20,7 @@ Ambari Agent
"""
from resource_management import *
+import os
def oozie():
import params
@@ -31,7 +32,7 @@ def oozie():
configuration_attributes=params.config['configuration_attributes']['oozie-site']
)
- File(format("{oozie_conf_dir}/oozie-env.cmd"),
+ File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_cmd_template)
)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py
index 8e677ef..47bb40f 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py
@@ -24,14 +24,14 @@ from resource_management import *
def pig():
import params
- File(format("{params.pig_conf_dir}/pig.properties"),
+ File(os.path.join(params.pig_conf_dir, "pig.properties"),
mode="f",
owner=params.pig_user,
content=params.pig_properties
)
if (params.log4j_props != None):
- File(format("{params.pig_conf_dir}/log4j.properties"),
+ File(os.path.join(params.pig_conf_dir, "log4j.properties"),
mode='f',
owner=params.pig_user,
content=params.log4j_props
@@ -44,6 +44,6 @@ def pig_TemplateConfig(name):
name = [name]
for x in name:
- TemplateConfig( format("{pig_conf_dir}/{x}"),
+ TemplateConfig(os.path.join(params.pig_conf_dir,x),
owner = params.hdfs_user
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/configuration/sqoop-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/configuration/sqoop-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/configuration/sqoop-site.xml
deleted file mode 100644
index eccd168..0000000
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/configuration/sqoop-site.xml
+++ /dev/null
@@ -1,156 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<!-- Put Sqoop-specific properties in this file. -->
-
-<configuration>
-
- <!--
- Set the value of this property to explicitly enable third-party
- ManagerFactory plugins.
-
- If this is not used, you can alternately specify a set of ManagerFactories
- in the $SQOOP_CONF_DIR/managers.d/ subdirectory. Each file should contain
- one or more lines like:
- manager.class.name[=/path/to/containing.jar]
-
- Files will be consulted in lexicographical order only if this property
- is unset.
- -->
- <!--
- <property>
- <name>sqoop.connection.factories</name>
- <value>com.cloudera.sqoop.manager.DefaultManagerFactory</value>
- <description>A comma-delimited list of ManagerFactory implementations
- which are consulted, in order, to instantiate ConnManager instances
- used to drive connections to databases.
- </description>
- </property>
- -->
-
- <!--
- Set the value of this property to enable third-party tools.
-
- If this is not used, you can alternately specify a set of ToolPlugins
- in the $SQOOP_CONF_DIR/tools.d/ subdirectory. Each file should contain
- one or more lines like:
- plugin.class.name[=/path/to/containing.jar]
-
- Files will be consulted in lexicographical order only if this property
- is unset.
- -->
- <!--
- <property>
- <name>sqoop.tool.plugins</name>
- <value></value>
- <description>A comma-delimited list of ToolPlugin implementations
- which are consulted, in order, to register SqoopTool instances which
- allow third-party tools to be used.
- </description>
- </property>
- -->
-
- <!--
- By default, the Sqoop metastore will auto-connect to a local embedded
- database stored in ~/.sqoop/. To disable metastore auto-connect, uncomment
- this next property.
- -->
- <!--
- <property>
- <name>sqoop.metastore.client.enable.autoconnect</name>
- <value>false</value>
- <description>If true, Sqoop will connect to a local metastore
- for job management when no other metastore arguments are
- provided.
- </description>
- </property>
- -->
-
- <!--
- The auto-connect metastore is stored in ~/.sqoop/. Uncomment
- these next arguments to control the auto-connect process with
- greater precision.
- -->
- <!--
- <property>
- <name>sqoop.metastore.client.autoconnect.url</name>
- <value>jdbc:hsqldb:file:/tmp/sqoop-meta/meta.db;shutdown=true</value>
- <description>The connect string to use when connecting to a
- job-management metastore. If unspecified, uses ~/.sqoop/.
- You can specify a different path here.
- </description>
- </property>
- <property>
- <name>sqoop.metastore.client.autoconnect.username</name>
- <value>SA</value>
- <description>The username to bind to the metastore.
- </description>
- </property>
- <property>
- <name>sqoop.metastore.client.autoconnect.password</name>
- <value></value>
- <description>The password to bind to the metastore.
- </description>
- </property>
- -->
-
- <!--
- For security reasons, by default your database password will not be stored in
- the Sqoop metastore. When executing a saved job, you will need to
- reenter the database password. Uncomment this setting to enable saved
- password storage. (INSECURE!)
- -->
- <!--
- <property>
- <name>sqoop.metastore.client.record.password</name>
- <value>true</value>
- <description>If true, allow saved passwords in the metastore.
- </description>
- </property>
- -->
-
-
- <!--
- SERVER CONFIGURATION: If you plan to run a Sqoop metastore on this machine,
- you should uncomment and set these parameters appropriately.
-
- You should then configure clients with:
- sqoop.metastore.client.autoconnect.url =
- jdbc:hsqldb:hsql://<server-name>:<port>/sqoop
- -->
- <!--
- <property>
- <name>sqoop.metastore.server.location</name>
- <value>/tmp/sqoop-metastore/shared.db</value>
- <description>Path to the shared metastore database files.
- If this is not set, it will be placed in ~/.sqoop/.
- </description>
- </property>
-
- <property>
- <name>sqoop.metastore.server.port</name>
- <value>16000</value>
- <description>Port that this metastore should listen on.
- </description>
- </property>
- -->
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/params.py
index 7197be3..8d1d59c 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/params.py
@@ -26,7 +26,7 @@ sqoop_home_dir = None
sqoop_conf_dir = "conf"
hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], ".."))
-
+sqoop_env_cmd_template = config['configurations']['sqoop-env']['content']
if os.environ.has_key("SQOOP_HOME"):
sqoop_home_dir = os.environ["SQOOP_HOME"]
sqoop_conf_dir = os.path.join(sqoop_home_dir, "conf")
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop.py
index a906dd2..f32c20e 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop.py
@@ -18,13 +18,10 @@ limitations under the License.
"""
from resource_management import *
-import sys
+import os
def sqoop():
import params
- XmlConfig("sqoop-site.xml",
- conf_dir=params.sqoop_conf_dir,
- configurations=params.config['configurations']['sqoop-site'],
- owner=params.sqoop_user,
- mode="f"
+ File(os.path.join(params.sqoop_conf_dir, "sqoop-env.cmd"),
+ content=InlineTemplate(params.sqoop_env_cmd_template)
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop_client.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop_client.py
index 34673fc..1aa5e01 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop_client.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/SQOOP/package/scripts/sqoop_client.py
@@ -20,14 +20,14 @@ limitations under the License.
import sys
from resource_management import *
-
-
+from sqoop import sqoop
class SqoopClient(Script):
def install(self, env):
- import params
- if params.sqoop_home_dir is None:
- self.install_packages(env)
+ import params
+ if params.sqoop_home_dir is None:
+ self.install_packages(env)
+ self.configure(env)
def configure(self, env):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/nimbus.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/nimbus.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/nimbus.py
index d9ca857..322ecfd 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/nimbus.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/nimbus.py
@@ -28,7 +28,7 @@ class Nimbus(Script):
def install(self, env):
if not check_windows_service_exists(service_mapping.nimbus_win_service_name):
self.install_packages(env)
- self.configure(env)
+ self.configure(env)
def start(self, env):
self.configure(env)
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/supervisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/supervisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/supervisor.py
index 827b3ba..079f51f 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/supervisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/supervisor.py
@@ -28,7 +28,7 @@ class Supervisor(Script):
def install(self, env):
if not check_windows_service_exists(service_mapping.supervisor_win_service_name):
self.install_packages(env)
- self.configure(env)
+ self.configure(env)
def start(self, env):
self.configure(env)
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/yaml_config.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/yaml_config.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/yaml_config.py
index 55aa777..a66f2a3 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/yaml_config.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/STORM/package/scripts/yaml_config.py
@@ -20,6 +20,7 @@ limitations under the License.
from resource_management import *
+import os
def escape_yaml_propetry(value):
unquouted = False
@@ -57,10 +58,11 @@ def yaml_config(
owner = None,
group = None
):
+ import params
config_content = source.InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_propetry(value) }}
{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_propetry])
- File (format("{conf_dir}/{filename}"),
+ File (os.path.join(params.conf_dir, filename),
content = config_content,
owner = owner,
mode = "f"
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py
index 6b4a5aa..4e3324e 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py
@@ -27,6 +27,7 @@ class TezClient(Script):
import params
if params.tez_home_dir is None:
self.install_packages(env)
+ self.configure(env)
def configure(self, env):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/package/scripts/zookeeper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/package/scripts/zookeeper.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/package/scripts/zookeeper.py
index 15bc881..2cacd57 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/package/scripts/zookeeper.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/package/scripts/zookeeper.py
@@ -35,12 +35,12 @@ def zookeeper(type = None):
recursive=True
)
if (params.log4j_props != None):
- File(format("{params.zookeeper_conf_dir}/log4j.properties"),
+ File(os.path.join(params.zookeeper_conf_dir, "log4j.properties"),
mode="f",
owner=params.zk_user,
content=params.log4j_props
)
- elif (os.path.exists(format("{params.zookeeper_conf_dir}/log4j.properties"))):
+ elif (os.path.exists(os.path.join(params.zookeeper_conf_dir, "log4j.properties"))):
File(format("{params.zookeeper_conf_dir}/log4j.properties"),
mode="f",
owner=params.zk_user
@@ -56,7 +56,7 @@ def zookeeper(type = None):
def configFile(name, template_name=None):
import params
- File(format("{params.zookeeper_conf_dir}/{name}"),
+ File(os.path.join(params.zookeeper_conf_dir, name),
content=Template(template_name),
owner=params.zk_user,
mode="f"
http://git-wip-us.apache.org/repos/asf/ambari/blob/f9a08f9f/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/package/scripts/zookeeper_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/package/scripts/zookeeper_client.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/package/scripts/zookeeper_client.py
index fa79b8e..663766f 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/package/scripts/zookeeper_client.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/ZOOKEEPER/package/scripts/zookeeper_client.py
@@ -29,6 +29,7 @@ class ZookeeperClient(Script):
# client checks env var to determine if it is installed
if not os.environ.has_key("ZOOKEEPER_HOME"):
self.install_packages(env)
+ self.configure(env)
def configure(self, env):
import params