You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vb...@apache.org on 2015/04/06 17:12:17 UTC
ambari git commit: Revert "AMBARI-10346. Ambari: When configured to
run with Oracle DB,
mysql-connector jar is found instead of ojdbc jar.(vbrodetskyi)"
Repository: ambari
Updated Branches:
refs/heads/trunk fc03a3928 -> 4d37ee7d1
Revert "AMBARI-10346. Ambari: When configured to run with Oracle DB, mysql-connector jar is found instead of ojdbc jar.(vbrodetskyi)"
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4d37ee7d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4d37ee7d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4d37ee7d
Branch: refs/heads/trunk
Commit: 4d37ee7d1849f79ad03e8dbfacaabf79871ba8c8
Parents: fc03a39
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Mon Apr 6 10:06:18 2015 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Mon Apr 6 10:06:18 2015 +0300
----------------------------------------------------------------------
ambari-server/src/main/python/ambari-server.py | 2 +-
.../python/ambari_server/dbConfiguration.py | 3 +-
.../SQOOP/1.4.4.2.0/configuration/sqoop-env.xml | 5 ++
.../SQOOP/1.4.4.2.0/package/scripts/params.py | 35 +++++++++++++
.../SQOOP/1.4.4.2.0/package/scripts/sqoop.py | 38 +++++++++++++-
.../python/stacks/2.0.6/SQOOP/test_sqoop.py | 52 ++++++++++++++++++++
6 files changed, 131 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4d37ee7d/ambari-server/src/main/python/ambari-server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py
index 8b23d83..0824268 100755
--- a/ambari-server/src/main/python/ambari-server.py
+++ b/ambari-server/src/main/python/ambari-server.py
@@ -357,7 +357,7 @@ def init_parser_options(parser):
parser.add_option('--jdbc-driver', default=None, help="Specifies the path to the JDBC driver JAR file for the " \
"database type specified with the --jdbc-db option. Used only with --jdbc-db option.",
dest="jdbc_driver")
- parser.add_option('--jdbc-db', default=None, help="Specifies the database type [postgres|mysql|oracle] for the " \
+ parser.add_option('--jdbc-db', default=None, help="Specifies the database type [postgres|mysql|oracle|hsqldb] for the " \
"JDBC driver specified with the --jdbc-driver option. Used only with --jdbc-driver option.",
dest="jdbc_db")
parser.add_option('--cluster-name', default=None, help="Cluster name", dest="cluster_name")
http://git-wip-us.apache.org/repos/asf/ambari/blob/4d37ee7d/ambari-server/src/main/python/ambari_server/dbConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/dbConfiguration.py b/ambari-server/src/main/python/ambari_server/dbConfiguration.py
index 2913e3f..5e26096 100644
--- a/ambari-server/src/main/python/ambari_server/dbConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/dbConfiguration.py
@@ -328,7 +328,8 @@ class DBMSConfigFactoryLinux(DBMSConfigFactory):
'oracle',
'mysql',
'postgres',
- 'mssql'
+ 'mssql',
+ 'hsqldb'
]
self.DBMS_LIST = [
http://git-wip-us.apache.org/repos/asf/ambari/blob/4d37ee7d/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml
index 9a740be..82fa9d0 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml
@@ -51,4 +51,9 @@ export SQOOP_USER_CLASSPATH="`ls ${HIVE_HOME}/lib/libthrift-*.jar 2> /dev/null`:
<property-type>USER</property-type>
<value>sqoop</value>
</property>
+ <property>
+ <name>jdbc_drivers</name>
+ <description>Comma separated list of additional JDBC drivers class names</description>
+ <value> </value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4d37ee7d/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py
index e12acbe..d5278c4 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py
@@ -22,6 +22,7 @@ from resource_management.libraries.functions.default import default
from resource_management import *
config = Script.get_config()
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
stack_name = default("/hostLevelParams/stack_name", None)
@@ -58,3 +59,37 @@ sqoop_user = config['configurations']['sqoop-env']['sqoop_user']
smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
kinit_path_local = functions.get_kinit_path()
+#JDBC driver jar name
+sqoop_jdbc_drivers_dict = {}
+sqoop_jdbc_drivers_name_dict = {}
+if "jdbc_drivers" in config['configurations']['sqoop-env']:
+ sqoop_jdbc_drivers = config['configurations']['sqoop-env']['jdbc_drivers'].split(',')
+
+ for driver_name in sqoop_jdbc_drivers:
+ driver_name = driver_name.strip()
+ if driver_name and not driver_name == '':
+ if driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver":
+ jdbc_jar_name = "sqljdbc4.jar"
+ jdbc_symlink_name = "mssql-jdbc-driver.jar"
+ jdbc_driver_name = "mssql"
+ elif driver_name == "com.mysql.jdbc.Driver":
+ jdbc_jar_name = "mysql-connector-java.jar"
+ jdbc_symlink_name = "mysql-jdbc-driver.jar"
+ jdbc_driver_name = "mysql"
+ elif driver_name == "org.postgresql.Driver":
+ jdbc_jar_name = "postgresql-jdbc.jar"
+ jdbc_symlink_name = "postgres-jdbc-driver.jar"
+ jdbc_driver_name = "postgres"
+ elif driver_name == "oracle.jdbc.driver.OracleDriver":
+ jdbc_jar_name = "ojdbc.jar"
+ jdbc_symlink_name = "oracle-jdbc-driver.jar"
+ jdbc_driver_name = "oracle"
+ elif driver_name == "org.hsqldb.jdbc.JDBCDriver":
+ jdbc_jar_name = "hsqldb.jar"
+ jdbc_symlink_name = "hsqldb-jdbc-driver.jar"
+ jdbc_driver_name = "hsqldb"
+ else:
+ continue
+ sqoop_jdbc_drivers_dict[jdbc_jar_name] = jdbc_symlink_name
+ sqoop_jdbc_drivers_name_dict[jdbc_jar_name] = jdbc_driver_name
+jdk_location = config['hostLevelParams']['jdk_location']
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/4d37ee7d/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
index 99ad575..ff1db89 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
@@ -17,14 +17,21 @@ limitations under the License.
"""
-from resource_management import *
+from resource_management.core.source import InlineTemplate, DownloadSource
+from resource_management.libraries.functions import format
+from resource_management.core.resources.system import File, Link, Directory
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from ambari_commons import OSConst
import os
def sqoop(type=None):
import params
Link(params.sqoop_lib + "/mysql-connector-java.jar",
to = '/usr/share/java/mysql-connector-java.jar'
- )
+ )
+
+ jdbc_connector()
+
Directory(params.sqoop_conf_dir,
owner = params.sqoop_user,
group = params.user_group,
@@ -49,3 +56,30 @@ def update_config_permissions(names):
group = params.user_group,
only_if = format("test -e {full_filename}")
)
+
+def jdbc_connector():
+ import params
+ from urllib2 import HTTPError
+ from resource_management import Fail
+ for jar_name in params.sqoop_jdbc_drivers_dict:
+ if 'mysql-connector-java.jar' in jar_name:
+ continue
+ downloaded_custom_connector = format("{sqoop_lib}/{jar_name}")
+ jdbc_symlink_remote = params.sqoop_jdbc_drivers_dict[jar_name]
+ jdbc_driver_label = params.sqoop_jdbc_drivers_name_dict[jar_name]
+ driver_curl_source = format("{jdk_location}/{jdbc_symlink_remote}")
+ environment = {
+ "no_proxy": format("{ambari_server_hostname}")
+ }
+ try:
+ File(downloaded_custom_connector,
+ content = DownloadSource(driver_curl_source),
+ mode = 0644,
+ )
+ except HTTPError:
+ error_string = format("Could not download {driver_curl_source}\n\
+ Please upload jdbc driver to server by run command:\n\
+ ambari-server setup --jdbc-db={jdbc_driver_label} --jdbc-driver=<PATH TO DRIVER>\n\
+ at {ambari_server_hostname}")
+ raise Fail(error_string)
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/4d37ee7d/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py b/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py
index 1fe98db..b65ada9 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py
@@ -19,6 +19,7 @@ limitations under the License.
'''
from mock.mock import MagicMock, call, patch
from stacks.utils.RMFTestCase import *
+import json
class TestSqoop(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "SQOOP/1.4.4.2.0/package"
@@ -57,6 +58,57 @@ class TestSqoop(RMFTestCase):
group = 'hadoop',)
self.assertNoMoreResources()
+ def test_configure_add_jdbc(self):
+ config_file = self._getSrcFolder()+"/test/python/stacks/2.0.6/configs/secured.json"
+ with open(config_file, "r") as f:
+ loaded_json = json.load(f)
+ loaded_json['configurations']['sqoop-env']['jdbc_drivers'] = 'org.postgresql.Driver, oracle.jdbc.driver.OracleDriver'
+
+
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/sqoop_client.py",
+ classname = "SqoopClient",
+ command = "configure",
+ config_dict = loaded_json,
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Link', '/usr/lib/sqoop/lib/mysql-connector-java.jar',
+ to = '/usr/share/java/mysql-connector-java.jar',
+ )
+ self.assertResourceCalled('File', '/usr/lib/sqoop/lib/ojdbc.jar',
+ content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//oracle-jdbc-driver.jar'),
+ mode = 0644,
+ )
+ self.assertResourceCalled('File', '/usr/lib/sqoop/lib/postgresql-jdbc.jar',
+ content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//postgres-jdbc-driver.jar'),
+ mode = 0644,
+ )
+ self.assertResourceCalled('Directory', '/usr/lib/sqoop/conf',
+ owner = 'sqoop',
+ group = 'hadoop',
+ recursive = True,
+ )
+ self.assertResourceCalled('File', '/usr/lib/sqoop/conf/sqoop-env.sh',
+ content = InlineTemplate(self.getConfig()['configurations']['sqoop-env']['content']),
+ owner = 'sqoop',
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('File', '/usr/lib/sqoop/conf/sqoop-env-template.sh',
+ owner = 'sqoop',
+ only_if = 'test -e /usr/lib/sqoop/conf/sqoop-env-template.sh',
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('File', '/usr/lib/sqoop/conf/sqoop-site-template.xml',
+ owner = 'sqoop',
+ only_if = 'test -e /usr/lib/sqoop/conf/sqoop-site-template.xml',
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('File', '/usr/lib/sqoop/conf/sqoop-site.xml',
+ owner = 'sqoop',
+ only_if = 'test -e /usr/lib/sqoop/conf/sqoop-site.xml',
+ group = 'hadoop',
+ )
+ self.assertNoMoreResources()