You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2016/08/01 21:25:53 UTC
ambari git commit: AMBARI-17955. Enabled download of berkeley DB jar
file to be provisioned during install (Venkat Ranganathan via alejandro)
Repository: ambari
Updated Branches:
refs/heads/branch-2.4 3f20c4005 -> 8d8e64fec
AMBARI-17955. Enabled download of berkeley DB jar file to be provisioned during install (Venkat Ranganathan via alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8d8e64fe
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8d8e64fe
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8d8e64fe
Branch: refs/heads/branch-2.4
Commit: 8d8e64fec79bbe890d22dabd388e8a4988dba486
Parents: 3f20c40
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Mon Aug 1 14:28:53 2016 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Mon Aug 1 14:29:16 2016 -0700
----------------------------------------------------------------------
.../python/ambari_server/dbConfiguration.py | 22 ++++++++-----
.../ambari_server/dbConfiguration_linux.py | 34 ++++++++++++++++++++
.../FALCON/0.5.0.2.1/package/scripts/falcon.py | 20 ++++++++++++
.../0.5.0.2.1/package/scripts/params_linux.py | 7 ++++
.../stacks/HDP/2.5/services/stack_advisor.py | 7 ++++
.../src/test/python/TestAmbariServer.py | 6 ++--
.../stacks/2.1/FALCON/test_falcon_server.py | 7 +++-
7 files changed, 91 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8d8e64fe/ambari-server/src/main/python/ambari_server/dbConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/dbConfiguration.py b/ambari-server/src/main/python/ambari_server/dbConfiguration.py
index fc68a4c..ede257f 100644
--- a/ambari-server/src/main/python/ambari_server/dbConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/dbConfiguration.py
@@ -40,11 +40,11 @@ SETUP_DB_CONNECT_ATTEMPTS = 3
USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$"
PASSWORD_PATTERN = "^[a-zA-Z0-9_-]*$"
-CUSTOM_JDBC_DB_NAMES = ["postgres", "mysql", "mssql", "oracle", "hsqldb", "sqlanywhere"]
-DATABASE_NAMES = ["postgres", "oracle", "mysql", "mssql", "sqlanywhere"]
+CUSTOM_JDBC_DB_NAMES = ["postgres", "mysql", "mssql", "oracle", "hsqldb", "sqlanywhere", "bdb"]
+DATABASE_NAMES = ["postgres", "oracle", "mysql", "mssql", "sqlanywhere", "bdb"]
DATABASE_FULL_NAMES = {"oracle": "Oracle", "mysql": "MySQL", "mssql": "Microsoft SQL Server", "postgres":
- "PostgreSQL", "sqlanywhere": "SQL Anywhere"}
-LINUX_DBMS_KEYS_LIST = [ 'embedded', 'oracle', 'mysql', 'postgres', 'mssql', 'sqlanywhere']
+ "PostgreSQL", "sqlanywhere": "SQL Anywhere", "bdb" : "Berkeley DB"}
+LINUX_DBMS_KEYS_LIST = [ 'embedded', 'oracle', 'mysql', 'postgres', 'mssql', 'sqlanywhere', 'bdb']
AMBARI_DATABASE_NAME = "ambari"
AMBARI_DATABASE_TITLE = "ambari"
@@ -55,7 +55,8 @@ default_connectors_map = { "mssql":"sqljdbc4.jar",
"postgres":"postgresql-jdbc.jar",
"oracle":"ojdbc.jar",
"sqlanywhere":"sajdbc4.jar",
- "hsqldb":"hsqldb.jar"}
+ "hsqldb":"hsqldb.jar",
+ "bdb": 'je-5.0.73.jar'}
STORAGE_TYPE_LOCAL = 'local'
STORAGE_TYPE_REMOTE = 'remote'
@@ -343,7 +344,7 @@ class DBMSConfigFactoryWindows(DBMSConfigFactory):
class DBMSConfigFactoryLinux(DBMSConfigFactory):
def __init__(self):
from ambari_server.dbConfiguration_linux import createPGConfig, createOracleConfig, createMySQLConfig, \
- createMSSQLConfig, createSQLAConfig
+ createMSSQLConfig, createSQLAConfig, createBDBConfig
self.DBMS_KEYS_LIST = LINUX_DBMS_KEYS_LIST
@@ -353,7 +354,8 @@ class DBMSConfigFactoryLinux(DBMSConfigFactory):
'postgres',
'mssql',
'hsqldb',
- 'sqlanywhere'
+ 'sqlanywhere',
+ 'bdb'
]
self.DBMS_LIST = [
@@ -362,7 +364,9 @@ class DBMSConfigFactoryLinux(DBMSConfigFactory):
DBMSDesc(self.DBMS_KEYS_LIST[2], STORAGE_TYPE_REMOTE, 'MySQL / MariaDB', '', createMySQLConfig),
DBMSDesc(self.DBMS_KEYS_LIST[3], STORAGE_TYPE_REMOTE, 'PostgreSQL', '', createPGConfig),
DBMSDesc(self.DBMS_KEYS_LIST[4], STORAGE_TYPE_REMOTE, 'Microsoft SQL Server', 'Tech Preview', createMSSQLConfig),
- DBMSDesc(self.DBMS_KEYS_LIST[5], STORAGE_TYPE_REMOTE, 'SQL Anywhere', '', createSQLAConfig)
+ DBMSDesc(self.DBMS_KEYS_LIST[5], STORAGE_TYPE_REMOTE, 'SQL Anywhere', '', createSQLAConfig),
+ DBMSDesc(self.DBMS_KEYS_LIST[6], STORAGE_TYPE_REMOTE, 'BDB', '', createBDBConfig)
+
]
self.DBMS_DICT = \
@@ -381,6 +385,8 @@ class DBMSConfigFactoryLinux(DBMSConfigFactory):
self.DBMS_KEYS_LIST[3] + '-' + STORAGE_TYPE_REMOTE: 3,
self.DBMS_KEYS_LIST[5] + '-': 5,
self.DBMS_KEYS_LIST[5] + '-' + STORAGE_TYPE_REMOTE: 5,
+ self.DBMS_KEYS_LIST[6] + '-': 6,
+ self.DBMS_KEYS_LIST[6] + '-' + STORAGE_TYPE_LOCAL: 6,
}
self.DBMS_PROMPT_PATTERN = "[{0}] - {1}{2}\n"
http://git-wip-us.apache.org/repos/asf/ambari/blob/8d8e64fe/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py b/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py
index d09bd77..c595e41 100644
--- a/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py
+++ b/ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py
@@ -1119,3 +1119,37 @@ class SQLAConfig(LinuxDBMSConfig):
def createSQLAConfig(options, properties, storage_type, dbId):
return SQLAConfig(options, properties, storage_type)
+
+class BDBConfig(LinuxDBMSConfig):
+ def __init__(self, options, properties, storage_type):
+ super(BDBConfig, self).__init__(options, properties, storage_type)
+
+ #Init the database configuration data here, if any
+ self.dbms = "bdb"
+ self.dbms_full_name = "Berkeley DB Jar file"
+ self.driver_class_name = "com.berkeleydb.Driver"
+ self.driver_file_name = "je-5.0.73.jar"
+ self.driver_symlink_name = "bdb-jdbc-driver.jar"
+
+ self.database_storage_name = "Database"
+ self.client_tool_usage_pattern = ''
+
+ #
+ # Private implementation
+ #
+
+
+ def _is_jdbc_driver_installed(self, properties):
+ return LinuxDBMSConfig._find_jdbc_driver("*je-*.jar")
+
+ def _get_jdbc_driver_path(self, properties):
+ super(BDBConfig, self)._get_jdbc_driver_path(properties)
+
+
+ def _configure_database_name(self):
+ self.database_name = LinuxDBMSConfig._get_validated_db_name(self.database_storage_name, self.database_name)
+ return True
+
+
+def createBDBConfig(options, properties, storage_type, dbId):
+ return BDBConfig(options, properties, storage_type)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8d8e64fe/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
index 17c3ccd..5712516 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
@@ -18,11 +18,13 @@ limitations under the License.
"""
import os.path
+import traceback
# Local Imports
from resource_management.core.environment import Environment
from resource_management.core.source import InlineTemplate
from resource_management.core.source import Template
+from resource_management.core.source import DownloadSource
from resource_management.core.resources import Execute
from resource_management.core.resources.service import Service
from resource_management.core.resources.service import ServiceConfig
@@ -196,6 +198,24 @@ def falcon(type, action = None, upgrade_type=None):
process_exists = format("ls {server_pid_file} && ps -p {pid}")
if action == 'start':
+ if not os.path.exists(params.target_jar_file):
+ try :
+ File(params.target_jar_file,
+ content = DownloadSource(params.bdb_resource_name))
+ except :
+ exc_msg = traceback.format_exc()
+ exception_message = format("Caught Exception while downloading {bdb_resource_name}:\n{exc_msg}")
+ Logger.error(exception_message)
+
+ if not os.path.isfile(params.target_jar_file) :
+ error_message = """
+If you are using bdb as the Falcon graph db store, please run
+ambari-server setup --jdbc-db=bdb --jdbc-driver=<path to je5.0.73.jar
+on the ambari server host. Otherwise falcon startup will fail.
+Otherwise please configure Falcon to use HBase as the backend as described
+in the Falcon documentation.
+"""
+ Logger.error(error_message)
try:
Execute(format('{falcon_home}/bin/falcon-start -port {falcon_port}'),
user = params.falcon_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/8d8e64fe/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
index 089fdb0..349ebf1 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
@@ -45,6 +45,7 @@ version = default("/commandParams/version", None)
stack_version_unformatted = status_params.stack_version_unformatted
stack_version_formatted = status_params.stack_version_formatted
upgrade_direction = default("/commandParams/upgrade_direction", None)
+jdk_location = config['hostLevelParams']['jdk_location']
# current host stack version
current_version = default("/hostLevelParams/current_version", None)
@@ -134,6 +135,7 @@ supports_falcon_extensions = (stack_version_formatted and check_stack_feature(St
local_data_mirroring_dir = format('{stack_root}/current/falcon-server/data-mirroring')
dfs_data_mirroring_dir = "/apps/data-mirroring"
+
########################################################
############# Atlas related params #####################
########################################################
@@ -169,6 +171,11 @@ default_fs = config['configurations']['core-site']['fs.defaultFS']
dfs_type = default("/commandParams/dfs_type", "")
+bdb_jar_name = "je-5.0.73.jar"
+bdb_resource_name = format("{jdk_location}/{bdb_jar_name}")
+target_jar_file = os.path.join(falcon_webinf_lib, bdb_jar_name)
+
+
import functools
#create partial functions with common arguments for every HdfsResource call
#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
http://git-wip-us.apache.org/repos/asf/ambari/blob/8d8e64fe/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 86f15dc..05c73be 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -86,6 +86,12 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
self.__atlasFalconHookClassName = "org.apache.atlas.falcon.service.AtlasService"
super(HDP25StackAdvisor,self).recommendFalconConfigurations(configurations, clusterData, services, hosts)
+ def recommendFalconConfigurations(self, configurations, clusterData, services, hosts):
+ # Set the classname for the Falcon hook to 2.5 classname = all other 2.3
+ # stack advisor changes are needed.
+ self.atlasFalconHookClassName = "org.apache.atlas.falcon.service.AtlasService"
+ super(HDP25StackAdvisor,self).recommendFalconConfigurations(configurations, clusterData, services, hosts)
+
def createComponentLayoutRecommendations(self, services, hosts):
parentComponentLayoutRecommendations = super(HDP25StackAdvisor, self).createComponentLayoutRecommendations(
services, hosts)
@@ -396,6 +402,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
"ATLAS": self.recommendAtlasConfigurations,
"RANGER_KMS": self.recommendRangerKMSConfigurations,
"STORM": self.recommendStormConfigurations,
+ "FALCON": self.recommendFalconConfigurations,
"OOZIE": self.recommendOozieConfigurations,
"SPARK2": self.recommendSpark2Configurations
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8d8e64fe/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index 279a172..a45a4bd 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -5331,7 +5331,7 @@ class TestAmbariServer(TestCase):
get_ambari_properties_mock.return_value = properties
get_ambari_properties_3_mock.side_effect = get_ambari_properties_2_mock.side_effect = [properties, properties2, properties2]
- isfile_mock.side_effect = [False, True, False, False]
+ isfile_mock.side_effect = [False, True, False, False, False]
try:
upgrade(args)
@@ -6550,7 +6550,7 @@ class TestAmbariServer(TestCase):
isdir_mock.return_value = True
- isfile_mock.side_effect = [True, False, False, False]
+ isfile_mock.side_effect = [True, False, False, False, False]
del args.database_index
del args.persistence_type
@@ -6572,7 +6572,7 @@ class TestAmbariServer(TestCase):
get_ambari_properties_mock.reset_mock()
os_symlink_mock.reset_mock()
- isfile_mock.side_effect = [False, False, False, False]
+ isfile_mock.side_effect = [False, False, False, False, False]
check_jdbc_drivers(args)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8d8e64fe/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
index 2dd0da9..9f9b774 100644
--- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
+++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
@@ -43,6 +43,10 @@ class TestFalconServer(RMFTestCase):
self.assert_configure_default()
+ self.assertResourceCalled('File', '/usr/lib/falcon/server/webapp/falcon/WEB-INF/lib/je-5.0.73.jar',
+ content=DownloadSource('http://c6401.ambari.apache.org:8080/resources//je-5.0.73.jar')
+ )
+
self.assertResourceCalled('Execute', '/usr/lib/falcon/bin/falcon-start -port 15000',
path = ['/usr/bin'],
user = 'falcon',
@@ -368,6 +372,7 @@ class TestFalconServer(RMFTestCase):
owner = 'falcon',
create_parents = True,
)
+
self.assertResourceCalled('Execute', '/usr/hdp/current/falcon-server/bin/falcon-start -port 15000',
environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client'},
path = ['/usr/hdp/current/hadoop-client/bin'],
@@ -375,7 +380,7 @@ class TestFalconServer(RMFTestCase):
not_if = 'ls /var/run/falcon/falcon.pid && ps -p ',
)
self.assertNoMoreResources()
-
+
@patch("resource_management.libraries.functions.security_commons.build_expectations")
@patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
@patch("resource_management.libraries.functions.security_commons.validate_security_config_properties")