You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2014/02/28 22:14:17 UTC
git commit: AMBARI-4894. Allow upgrade stack to take URL as a
parameter (ncole)
Repository: ambari
Updated Branches:
refs/heads/trunk 060ae2cdb -> 08dc84acb
AMBARI-4894. Allow upgrade stack to take URL as a parameter (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/08dc84ac
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/08dc84ac
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/08dc84ac
Branch: refs/heads/trunk
Commit: 08dc84acb4e29fdd657a9f0d6937005ef8e5ece7
Parents: 060ae2c
Author: Nate Cole <nc...@hortonworks.com>
Authored: Fri Feb 28 15:26:11 2014 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Fri Feb 28 16:13:55 2014 -0500
----------------------------------------------------------------------
.../server/upgrade/StackUpgradeHelper.java | 49 +++++---
.../ambari/server/upgrade/StackUpgradeUtil.java | 59 ++++++++-
ambari-server/src/main/python/ambari-server.py | 20 ++-
.../server/upgrade/StackUpgradeUtilTest.java | 123 +++++++++++++++++++
.../src/test/python/TestAmbariServer.py | 48 +++++++-
5 files changed, 269 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/08dc84ac/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java
index b9fec20..77ec069 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeHelper.java
@@ -17,25 +17,28 @@
*/
package org.apache.ambari.server.upgrade;
-import com.google.gson.Gson;
-import com.google.inject.Guice;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.persist.PersistService;
-import com.google.inject.persist.Transactional;
-import org.apache.ambari.server.controller.ControllerModule;
-import org.apache.ambari.server.orm.DBAccessor;
-import org.apache.ambari.server.orm.dao.MetainfoDAO;
-import org.apache.ambari.server.orm.entities.MetainfoEntity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.InputMismatchException;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
+
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.ambari.server.controller.ControllerModule;
+import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.orm.dao.MetainfoDAO;
+import org.apache.ambari.server.orm.entities.MetainfoEntity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.Gson;
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.persist.PersistService;
+import com.google.inject.persist.Transactional;
public class StackUpgradeHelper {
private static final Logger LOG = LoggerFactory.getLogger
@@ -88,14 +91,18 @@ public class StackUpgradeHelper {
/**
* Change the stack id in the Ambari DB.
- * @param stackId
+ * @param stackInfo
* @throws SQLException
*/
- private void updateStackVersion(Map<String, String> stackId) throws SQLException {
- if (stackId == null || stackId.isEmpty()) {
- throw new IllegalArgumentException("Empty stack id. " + stackId);
+ public void updateStackVersion(Map<String, String> stackInfo) throws Exception {
+ if (stackInfo == null || stackInfo.isEmpty()) {
+ throw new IllegalArgumentException("Empty stack id. " + stackInfo);
}
- Iterator<Map.Entry<String, String>> stackIdEntry = stackId.entrySet().iterator();
+
+ String repoUrl = stackInfo.remove("repo_url");
+ String repoUrlOs = stackInfo.remove("repo_url_os");
+
+ Iterator<Map.Entry<String, String>> stackIdEntry = stackInfo.entrySet().iterator();
Map.Entry<String, String> stackEntry = stackIdEntry.next();
String stackName = stackEntry.getKey();
@@ -105,6 +112,10 @@ public class StackUpgradeHelper {
"stackVersion = "+ stackVersion);
stackUpgradeUtil.updateStackDetails(stackName, stackVersion);
+
+ if (null != repoUrl) {
+ stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, repoUrl, repoUrlOs);
+ }
dbAccessor.updateTable("hostcomponentstate", "current_state", "INSTALLED", "where current_state = 'UPGRADING'");
}
@@ -139,11 +150,13 @@ public class StackUpgradeHelper {
"actions: " + stackUpgradeHelper.getValidActions());
}
+
stackUpgradeHelper.startPersistenceService();
Map values = gson.fromJson(valueMap, Map.class);
if (action.equals(STACK_ID_UPDATE_ACTION)) {
stackUpgradeHelper.updateStackVersion(values);
+
} else if (action.equals(METAINFO_UPDATE_ACTION)) {
stackUpgradeHelper.updateMetaInfo(values);
http://git-wip-us.apache.org/repos/asf/ambari/blob/08dc84ac/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeUtil.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeUtil.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeUtil.java
index 75189cc..55697dc 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeUtil.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/StackUpgradeUtil.java
@@ -17,10 +17,14 @@
*/
package org.apache.ambari.server.upgrade;
-import com.google.gson.Gson;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.persist.Transactional;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Scanner;
+import java.util.Set;
+
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.orm.dao.ClusterDAO;
import org.apache.ambari.server.orm.dao.ClusterStateDAO;
import org.apache.ambari.server.orm.dao.HostComponentDesiredStateDAO;
@@ -33,9 +37,13 @@ import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity;
import org.apache.ambari.server.orm.entities.HostComponentStateEntity;
import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity;
import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntity;
+import org.apache.ambari.server.state.OperatingSystemInfo;
import org.apache.ambari.server.state.StackId;
-import java.util.ArrayList;
-import java.util.List;
+
+import com.google.gson.Gson;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.persist.Transactional;
public class StackUpgradeUtil {
@Inject
@@ -139,4 +147,43 @@ public class StackUpgradeUtil {
}
+
+ /**
+ * @param stackName
+ * @param stackVersion
+ * @param localRepo
+ */
+ public void updateLocalRepo(String stackName, String stackVersion,
+ String repoUrl, String repoUrlOs) throws Exception {
+
+ if (null == repoUrl ||
+ repoUrl.isEmpty() ||
+ !repoUrl.startsWith("http"))
+ return;
+
+ String server = repoUrl;
+
+ String[] oses = new String[0];
+
+ if (null != repoUrlOs) {
+ oses = repoUrlOs.split(",");
+ }
+
+ AmbariMetaInfo ami = injector.getInstance(AmbariMetaInfo.class);
+
+ if (0 == oses.length) {
+ // do them all
+ for (OperatingSystemInfo osi : ami.getOperatingSystems(stackName, stackVersion)) {
+ ami.updateRepoBaseURL(stackName, stackVersion, osi.getOsType(),
+ stackName + "-" + stackVersion, server);
+ }
+
+ } else {
+ for (String os : oses) {
+ ami.updateRepoBaseURL(stackName, stackVersion, os,
+ stackName + "-" + stackVersion, server);
+ }
+ }
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/08dc84ac/ambari-server/src/main/python/ambari-server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py
index 30b778c..c4fa246 100755
--- a/ambari-server/src/main/python/ambari-server.py
+++ b/ambari-server/src/main/python/ambari-server.py
@@ -2487,7 +2487,7 @@ def stop(args):
### Stack upgrade ###
-def upgrade_stack(args, stack_id):
+def upgrade_stack(args, stack_id, repo_url = None, repo_url_os = None):
if not is_root():
err = 'Ambari-server upgradestack should be run with ' \
'root-level privileges'
@@ -2495,7 +2495,7 @@ def upgrade_stack(args, stack_id):
check_database_name_property()
stack_name, stack_version = stack_id.split(STACK_NAME_VER_SEP)
- retcode = run_stack_upgrade(stack_name, stack_version)
+ retcode = run_stack_upgrade(stack_name, stack_version, repo_url, repo_url_os)
if not retcode == 0:
raise FatalException(retcode, 'Stack upgrade failed.')
@@ -2606,7 +2606,7 @@ def run_schema_upgrade(version):
print_error_msg("Error executing schema upgrade, please check the server logs.")
return retcode
-def run_stack_upgrade(stackName, stackVersion):
+def run_stack_upgrade(stackName, stackVersion, repo_url, repo_url_os):
jdk_path = find_jdk()
if jdk_path is None:
print_error_msg("No JDK found, please run the \"setup\" "
@@ -2615,6 +2615,10 @@ def run_stack_upgrade(stackName, stackVersion):
return 1
stackId = {}
stackId[stackName] = stackVersion
+ if repo_url is not None:
+ stackId['repo_url'] = repo_url
+ if repo_url_os is not None:
+ stackId['repo_url_os'] = repo_url_os
command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_conf_dir(), get_ambari_classpath(),
"updateStackId",
@@ -4070,7 +4074,15 @@ def main():
upgrade(options)
elif action == UPGRADE_STACK_ACTION:
stack_id = args[1]
- upgrade_stack(options, stack_id)
+ repo_url = None
+ repo_url_os = None
+
+ if len(args) > 2:
+ repo_url = args[2]
+ if len(args) > 3:
+ repo_url_os = args[3]
+
+ upgrade_stack(options, stack_id, repo_url, repo_url_os)
elif action == LDAP_SETUP_ACTION:
setup_ldap()
elif action == SETUP_SECURITY_ACTION:
http://git-wip-us.apache.org/repos/asf/ambari/blob/08dc84ac/ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeUtilTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeUtilTest.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeUtilTest.java
new file mode 100644
index 0000000..5cda29c
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/StackUpgradeUtilTest.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.upgrade;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.orm.GuiceJpaInitializer;
+import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
+import org.apache.ambari.server.orm.dao.MetainfoDAO;
+import org.apache.ambari.server.orm.entities.MetainfoEntity;
+import org.apache.ambari.server.state.RepositoryInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.persist.PersistService;
+
+/**
+ * Tests the StackUpgradeHelper
+ */
+public class StackUpgradeUtilTest {
+
+ private Injector injector;
+
+ @Before
+ public void setup() throws Exception {
+ injector = Guice.createInjector(new InMemoryDefaultTestModule());
+ injector.getInstance(GuiceJpaInitializer.class);
+ }
+
+ @After
+ public void teardown() throws Exception {
+ injector.getInstance(PersistService.class).stop();
+ }
+
+ private void reset(String stackName, String stackVersion) throws Exception {
+ AmbariMetaInfo ami = injector.getInstance(AmbariMetaInfo.class);
+
+ for (Entry<String, List<RepositoryInfo>> entry : ami.getRepository(stackName, stackVersion).entrySet()) {
+ for (RepositoryInfo ri : entry.getValue()) {
+ if (-1 == ri.getRepoId().indexOf("epel")) {
+ ami.updateRepoBaseURL(stackName, stackVersion,
+ ri.getOsType(), ri.getRepoId(), ri.getDefaultBaseUrl());
+ }
+ }
+ }
+
+ }
+
+ @Test
+ public void testUpgradeStack() throws Exception {
+ StackUpgradeUtil stackUpgradeUtil = injector.getInstance(StackUpgradeUtil.class);
+
+ String stackName = "HDP";
+ String stackVersion = "1.3.0";
+ String localRepoUrl = "http://foo.bar";
+
+ // check updating all
+ stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, localRepoUrl, null);
+
+ MetainfoDAO dao = injector.getInstance(MetainfoDAO.class);
+
+ Collection<MetainfoEntity> entities = dao.findAll();
+ Assert.assertTrue(entities.size() > 0);
+
+ for (MetainfoEntity entity : entities) {
+ Assert.assertTrue(entity.getMetainfoName().startsWith("repo:/HDP/1.3.0/"));
+ Assert.assertEquals(localRepoUrl, entity.getMetainfoValue());
+ }
+
+ reset (stackName, stackVersion);
+ entities = dao.findAll();
+ Assert.assertTrue(0 == entities.size());
+
+ // check updating only centos6
+ stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, localRepoUrl, "centos6");
+
+ entities = dao.findAll();
+ for (MetainfoEntity entity : entities) {
+ Assert.assertTrue(entity.getMetainfoName().startsWith("repo:/HDP/1.3.0/centos6"));
+ Assert.assertEquals(localRepoUrl, entity.getMetainfoValue());
+ }
+
+ reset (stackName, stackVersion);
+ entities = dao.findAll();
+ Assert.assertTrue(0 == entities.size());
+
+ // check updating only centos6 and centos5
+ stackUpgradeUtil.updateLocalRepo(stackName, stackVersion, localRepoUrl, "centos6,centos5");
+
+ entities = dao.findAll();
+ for (MetainfoEntity entity : entities) {
+ Assert.assertTrue(entity.getMetainfoName().startsWith("repo:/HDP/1.3.0/centos6") ||
+ entity.getMetainfoName().startsWith("repo:/HDP/1.3.0/centos5"));
+ Assert.assertEquals(localRepoUrl, entity.getMetainfoValue());
+ }
+
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/08dc84ac/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index d1053aa..ca85b77 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -2740,7 +2740,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
ambari_server.upgrade_stack(args, 'HDP-2.0')
self.assertTrue(run_stack_upgrade_mock.called)
- run_stack_upgrade_mock.assert_called_with("HDP", "2.0")
+ run_stack_upgrade_mock.assert_called_with("HDP", "2.0", None, None)
@patch.object(ambari_server, 'get_conf_dir')
@patch.object(ambari_server, 'get_ambari_classpath')
@@ -2754,7 +2754,51 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
get_conf_dir_mock.return_value = '/etc/conf'
stackIdMap = {'HDP' : '2.0'}
- ambari_server.run_stack_upgrade('HDP', '2.0')
+ ambari_server.run_stack_upgrade('HDP', '2.0', None, None)
+
+ self.assertTrue(jdk_path_mock.called)
+ self.assertTrue(get_ambari_classpath_mock.called)
+ self.assertTrue(get_conf_dir_mock.called)
+ self.assertTrue(run_os_command_mock.called)
+ run_os_command_mock.assert_called_with('/usr/lib/java/bin/java -cp /etc/conf:test:path12 '
+ 'org.apache.ambari.server.upgrade.StackUpgradeHelper '
+ 'updateStackId ' + json.dumps(stackIdMap) + ' > /var/log/ambari-server/ambari-server.out 2>&1')
+
+ @patch.object(ambari_server, 'get_conf_dir')
+ @patch.object(ambari_server, 'get_ambari_classpath')
+ @patch.object(ambari_server, 'run_os_command')
+ @patch.object(ambari_server, 'find_jdk')
+ def test_run_stack_upgrade_with_url(self, jdk_path_mock, run_os_command_mock,
+ get_ambari_classpath_mock, get_conf_dir_mock):
+ jdk_path_mock.return_value = "/usr/lib/java"
+ run_os_command_mock.return_value = (0, None, None)
+ get_ambari_classpath_mock.return_value = 'test:path12'
+ get_conf_dir_mock.return_value = '/etc/conf'
+ stackIdMap = {'HDP' : '2.0', 'repo_url' : 'http://test.com'}
+
+ ambari_server.run_stack_upgrade('HDP', '2.0', 'http://test.com', None)
+
+ self.assertTrue(jdk_path_mock.called)
+ self.assertTrue(get_ambari_classpath_mock.called)
+ self.assertTrue(get_conf_dir_mock.called)
+ self.assertTrue(run_os_command_mock.called)
+ run_os_command_mock.assert_called_with('/usr/lib/java/bin/java -cp /etc/conf:test:path12 '
+ 'org.apache.ambari.server.upgrade.StackUpgradeHelper '
+ 'updateStackId ' + json.dumps(stackIdMap) + ' > /var/log/ambari-server/ambari-server.out 2>&1')
+
+ @patch.object(ambari_server, 'get_conf_dir')
+ @patch.object(ambari_server, 'get_ambari_classpath')
+ @patch.object(ambari_server, 'run_os_command')
+ @patch.object(ambari_server, 'find_jdk')
+ def test_run_stack_upgrade_with_url_os(self, jdk_path_mock, run_os_command_mock,
+ get_ambari_classpath_mock, get_conf_dir_mock):
+ jdk_path_mock.return_value = "/usr/lib/java"
+ run_os_command_mock.return_value = (0, None, None)
+ get_ambari_classpath_mock.return_value = 'test:path12'
+ get_conf_dir_mock.return_value = '/etc/conf'
+ stackIdMap = {'HDP' : '2.0', 'repo_url': 'http://test.com', 'repo_url_os': 'centos5,centos6'}
+
+ ambari_server.run_stack_upgrade('HDP', '2.0', 'http://test.com', 'centos5,centos6')
self.assertTrue(jdk_path_mock.called)
self.assertTrue(get_ambari_classpath_mock.called)