You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2013/12/03 21:51:32 UTC
git commit: AMBARI-3909. Upgrade local repo definitions when
upgrading Ambari. (ncole)
Updated Branches:
refs/heads/branch-1.4.2 f0b798e32 -> d5fdf4714
AMBARI-3909. Upgrade local repo definitions when upgrading Ambari. (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d5fdf471
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d5fdf471
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d5fdf471
Branch: refs/heads/branch-1.4.2
Commit: d5fdf4714650cfb04bf777e65d0ffd37122853f2
Parents: f0b798e
Author: Nate Cole <nc...@hortonworks.com>
Authored: Tue Dec 3 15:04:01 2013 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Tue Dec 3 15:13:54 2013 -0500
----------------------------------------------------------------------
ambari-server/src/main/python/ambari-server.py | 140 +++++++++++++++++++
.../dml/Ambari-DML-Oracle-INSERT_METAINFO.sql | 24 ++++
.../dml/Ambari-DML-Postgres-INSERT_METAINFO.sql | 22 +++
.../src/test/python/TestAmbariServer.py | 73 +++++++++-
4 files changed, 258 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d5fdf471/ambari-server/src/main/python/ambari-server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py
index d2a9861..7b78a08 100755
--- a/ambari-server/src/main/python/ambari-server.py
+++ b/ambari-server/src/main/python/ambari-server.py
@@ -226,6 +226,7 @@ UPGRADE_STACK_CMD = ['su', 'postgres',
'--command=psql -f {0} -v stack_name="\'{1}\'" -v stack_version="\'{2}\'" -v dbname="{3}"']
UPDATE_METAINFO_CMD = 'curl -X PUT "http://{0}:{1}/api/v1/stacks2" -u "{2}":"{3}"'
+
PG_ERROR_BLOCKED = "is being accessed by other users"
PG_STATUS_RUNNING = "running"
PG_DEFAULT_PASSWORD = "bigdata"
@@ -377,6 +378,15 @@ JCE_MIN_FILESIZE = 5000
DEFAULT_DB_NAME = "ambari"
+# stack repo upgrade
+STACK_LOCATION_KEY = 'metadata.path'
+STACK_LOCATION_DEFAULT = '/var/lib/ambari-server/resources/stacks'
+DATABASE_INSERT_METAINFO_SCRIPTS = ['/var/lib/ambari-server/resources/upgrade/dml/Ambari-DML-Postgres-INSERT_METAINFO.sql',
+ '/var/lib/ambari-server/resources/upgrade/dml/Ambari-DML-Oracle-INSERT_METAINFO.sql',
+ '/var/lib/ambari-server/resources/upgrade/dml/Ambari-DML-MySQL-INSERT_METAINFO.sql']
+INSERT_METAINFO_CMD = ['su', 'postgres',
+ '--command=psql -f {0} -v metainfo_key="\'{1}\'" -v metainfo_value="\'{2}\'" -v dbname="{3}"']
+
#Apache License Header
ASF_LICENSE_HEADER = '''
# Copyright 2011 The Apache Software Foundation
@@ -2425,6 +2435,134 @@ def upgrade_stack(args, stack_id):
print_warning_msg(errdata)
return retcode
+def load_stack_values(version, filename):
+ import xml.etree.ElementTree as ET
+ values = {}
+ root = ET.parse(filename).getroot()
+ for ostag in root:
+ ostype = ostag.attrib['type']
+ for repotag in ostag:
+ reponametag = repotag.find('reponame')
+ repoidtag = repotag.find('repoid')
+ baseurltag = repotag.find('baseurl')
+ if reponametag is not None and repoidtag is not None and baseurltag is not None:
+ key = "repo:/" + reponametag.text
+ key += "/" + version
+ key += "/" + ostype
+ key += "/" + repoidtag.text
+ key += ":baseurl"
+ values[key] = baseurltag.text
+
+ return values
+
+def upgrade_local_repo_remote_db(args, sqlfile, dbkey, dbvalue):
+ tool = get_db_cli_tool(args)
+ if not tool:
+ args.warnings.append('{0} not found. Please, run DDL script manually'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX]))
+ if VERBOSE:
+ print_warning_msg('{0} not found'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX]))
+ return -1, "Client wasn't found", "Client wasn't found"
+
+ #TODO add support of other databases with scripts
+ if args.database == "oracle":
+ sid_or_sname = "sid"
+ if (hasattr(args, 'sid_or_sname') and args.sid_or_sname == "sname") or \
+ (hasattr(args, 'jdbc_url') and args.jdbc_url and re.match(ORACLE_SNAME_PATTERN, args.jdbc_url)):
+ print_info_msg("using SERVICE_NAME instead of SID for Oracle")
+ sid_or_sname = "service_name"
+
+ retcode, out, err = run_in_shell('{0} {1}'.format(tool, ORACLE_UPGRADE_STACK_ARGS.format(
+ args.database_username,
+ args.database_password,
+ args.database_host,
+ args.database_port,
+ args.database_name,
+ sqlfile,
+ sid_or_sname,
+ dbkey,
+ dbvalue
+ )))
+ return retcode, out, err
+
+ return -2, "Wrong database", "Wrong database"
+ pass
+
+def upgrade_local_repo_db(args, dbkey, dbvalue):
+ if not is_root():
+ err = 'Ambari-server upgrade_local_repo_db should be run with ' \
+ 'root-level privileges'
+ raise FatalException(4, err)
+ check_database_name_property()
+
+ parse_properties_file(args)
+ if args.persistence_type == "remote":
+ client_desc = DATABASE_NAMES[DATABASE_INDEX] + ' ' + DATABASE_CLI_TOOLS_DESC[DATABASE_INDEX]
+ client_usage_cmd = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_INSERT_METAINFO_SCRIPTS[DATABASE_INDEX], args.database_username,
+ BLIND_PASSWORD, args.database_name)
+ #TODO temporarty code
+ if not args.database == "oracle":
+ raise FatalException(-20, "Upgrade for remote database only supports Oracle.")
+
+ if get_db_cli_tool(args):
+ retcode, out, err = upgrade_local_repo_remote_db(args, DATABASE_INSERT_METAINFO_SCRIPTS[DATABASE_INDEX],
+ dbkey, dbvalue)
+ if not retcode == 0:
+ raise NonFatalException(err)
+
+ else:
+ err = 'Cannot find ' + client_desc + ' client in the path to upgrade the local ' + \
+ 'repo information.'
+ raise NonFatalException(err)
+
+ pass
+ else:
+ #password access to ambari-server and mapred
+ configure_database_username_password(args)
+ dbname = args.database_name
+ sqlfile = DATABASE_INSERT_METAINFO_SCRIPTS[0]
+ command = INSERT_METAINFO_CMD[:]
+ command[-1] = command[-1].format(sqlfile, dbkey, dbvalue, dbname)
+ retcode, outdata, errdata = run_os_command(command)
+ if not retcode == 0:
+ raise FatalException(retcode, errdata)
+ if errdata:
+ print_warning_msg(errdata)
+ return retcode
+ pass
+
+def upgrade_local_repo(args):
+ properties = get_ambari_properties()
+ if properties == -1:
+ print_error_msg ("Error getting ambari properties")
+ return -1
+
+ stack_location = properties[STACK_LOCATION_KEY]
+ if stack_location is None:
+ stack_location = STACK_LOCATION_DEFAULT
+
+ stack_root_local = os.path.join(stack_location, "HDPLocal")
+ if not os.path.exists(stack_root_local):
+ print_info_msg("HDPLocal stack directory does not exist, skipping")
+ return
+
+ stack_root = os.path.join(stack_location, "HDP")
+ if not os.path.exists(stack_root):
+ print_info_msg("HDP stack directory does not exist, skipping")
+ return
+
+ for stack_version_local in os.listdir(stack_root_local):
+ repo_file_local = os.path.join(stack_root_local, stack_version_local, "repos", "repoinfo.xml")
+ repo_file = os.path.join(stack_root, stack_version_local, "repos", "repoinfo.xml")
+
+ if os.path.exists(repo_file_local) and os.path.exists(repo_file):
+ local_values = load_stack_values(stack_version_local, repo_file_local)
+ repo_values = load_stack_values(stack_version_local, repo_file)
+ for k, v in local_values.iteritems():
+ if repo_values.has_key(k):
+ local_url = local_values[k]
+ repo_url = repo_values[k]
+ if repo_url != local_url:
+ upgrade_local_repo_db(args, k, local_url)
#
# Upgrades the Ambari Server.
@@ -2518,6 +2656,8 @@ def upgrade(args):
else:
adjust_directory_permissions(user)
+ # local repo
+ upgrade_local_repo(args)
#
# The Ambari Server status.
http://git-wip-us.apache.org/repos/asf/ambari/blob/d5fdf471/ambari-server/src/main/resources/upgrade/dml/Ambari-DML-Oracle-INSERT_METAINFO.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/dml/Ambari-DML-Oracle-INSERT_METAINFO.sql b/ambari-server/src/main/resources/upgrade/dml/Ambari-DML-Oracle-INSERT_METAINFO.sql
new file mode 100644
index 0000000..c0f269e
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade/dml/Ambari-DML-Oracle-INSERT_METAINFO.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- Update stack_name and stack_version
+-- &1 - metainfo_key
+-- &2 - metainfo_value
+
+INSERT INTO ambari.metainfo (metainfo_key, metainfo_value) SELECT "&1", "&2" FROM dual WHERE
+ NOT EXISTS (SELECT metainfo_key FROM ambari.metainfo WHERE metainfo_key = "&1")
http://git-wip-us.apache.org/repos/asf/ambari/blob/d5fdf471/ambari-server/src/main/resources/upgrade/dml/Ambari-DML-Postgres-INSERT_METAINFO.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/dml/Ambari-DML-Postgres-INSERT_METAINFO.sql b/ambari-server/src/main/resources/upgrade/dml/Ambari-DML-Postgres-INSERT_METAINFO.sql
new file mode 100644
index 0000000..068b4e0
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade/dml/Ambari-DML-Postgres-INSERT_METAINFO.sql
@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+\connect :dbname;
+
+INSERT INTO ambari.metainfo (metainfo_key, metainfo_value)
+ SELECT :metainfo_key, :metainfo_value
+ WHERE NOT EXISTS (SELECT metainfo_key FROM ambari.metainfo WHERE metainfo_key = :metainfo_key)
http://git-wip-us.apache.org/repos/asf/ambari/blob/d5fdf471/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index 352363b..13d2282 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -2695,7 +2695,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
@patch.object(ambari_server, "get_ambari_properties")
@patch.object(ambari_server, "get_db_cli_tool")
@patch.object(ambari_server, "execute_remote_script")
- def test_upgrade(self, execute_remote_script_mock, get_db_cli_tool_mock, get_ambari_properties_mock, is_root_mock,
+ @patch.object(ambari_server, "upgrade_local_repo")
+ def test_upgrade(self, upgrade_local_repo_mock, execute_remote_script_mock,
+ get_db_cli_tool_mock, get_ambari_properties_mock, is_root_mock,
parse_properties_file_mock,
update_ambari_properties_mock,
check_postgre_up_mock, execute_db_script_mock,
@@ -2733,6 +2735,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
self.assertTrue(print_warning_msg_mock.called)
warning_args = print_warning_msg_mock.call_args[0][0]
self.assertTrue("custom ambari user" in warning_args)
+ self.assertTrue(upgrade_local_repo_mock.called)
# Testing with defined custom user
read_ambari_user_mock.return_value = "ambari-custom-user"
@@ -4460,4 +4463,72 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
self.assertTrue(get_validated_string_input_mock.called)
self.assertEqual(get_validated_string_input_mock.call_count, 2)
+ @patch("os.listdir")
+ @patch("os.path.exists")
+ @patch.object(ambari_server, "load_stack_values")
+ @patch.object(ambari_server, "get_ambari_properties")
+ @patch.object(ambari_server, "upgrade_local_repo_db")
+ def test_upgrade_local_repo(self,
+ upgrade_local_repo_db_mock,
+ get_ambari_properties_mock,
+ load_stack_values_mock,
+ os_path_exists_mock,
+ os_listdir_mock):
+
+ from mock.mock import call
+ args = MagicMock()
+ args.persistence_type = "local"
+
+ def load_values_side_effect(*args, **kwargs):
+ res = {}
+ res['a'] = 'http://oldurl'
+ if -1 != args[1].find("HDPLocal"):
+ res['a'] = 'http://newurl'
+ return res
+
+ load_stack_values_mock.side_effect = load_values_side_effect
+
+ properties = ambari_server.Properties()
+ get_ambari_properties_mock.return_value = properties
+ os_path_exists_mock.return_value = 1
+ os_listdir_mock.return_value = ['1.1']
+
+ ambari_server.upgrade_local_repo(args)
+
+ self.assertTrue(get_ambari_properties_mock.called)
+ self.assertTrue(load_stack_values_mock.called)
+ self.assertTrue(upgrade_local_repo_db_mock.called)
+ @patch("os.listdir")
+ @patch("os.path.exists")
+ @patch.object(ambari_server, "load_stack_values")
+ @patch.object(ambari_server, "get_ambari_properties")
+ @patch.object(ambari_server, "upgrade_local_repo_db")
+ def test_upgrade_local_repo_nochange(self,
+ upgrade_local_repo_db_mock,
+ get_ambari_properties_mock,
+ load_stack_values_mock,
+ os_path_exists_mock,
+ os_listdir_mock):
+
+ from mock.mock import call
+ args = MagicMock()
+ args.persistence_type = "local"
+
+ def load_values_side_effect(*args, **kwargs):
+ res = {}
+ res['a'] = 'http://oldurl'
+ return res
+
+ load_stack_values_mock.side_effect = load_values_side_effect
+
+ properties = ambari_server.Properties()
+ get_ambari_properties_mock.return_value = properties
+ os_path_exists_mock.return_value = 1
+ os_listdir_mock.return_value = ['1.1']
+
+ ambari_server.upgrade_local_repo(args)
+
+ self.assertTrue(get_ambari_properties_mock.called)
+ self.assertTrue(load_stack_values_mock.called)
+ self.assertFalse(upgrade_local_repo_db_mock.called)