You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2013/06/14 19:49:32 UTC
svn commit: r1493187 - in /incubator/ambari/trunk:
ambari-agent/src/test/python/ ambari-server/sbin/
ambari-server/src/main/java/org/apache/ambari/server/api/services/
ambari-server/src/main/java/org/apache/ambari/server/controller/
ambari-server/src/m...
Author: swagle
Date: Fri Jun 14 17:49:31 2013
New Revision: 1493187
URL: http://svn.apache.org/r1493187
Log:
AMBARI-2112. Agent should not cache repo info to avoid situations of incorrect repo. (Dmitry Sen via swagle)
Modified:
incubator/ambari/trunk/ambari-agent/src/test/python/TestPuppetExecutor.py
incubator/ambari/trunk/ambari-server/sbin/ambari-server
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackResourceProvider.java
incubator/ambari/trunk/ambari-server/src/main/python/ambari-server.py
incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
Modified: incubator/ambari/trunk/ambari-agent/src/test/python/TestPuppetExecutor.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-agent/src/test/python/TestPuppetExecutor.py?rev=1493187&r1=1493186&r2=1493187&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-agent/src/test/python/TestPuppetExecutor.py (original)
+++ incubator/ambari/trunk/ambari-agent/src/test/python/TestPuppetExecutor.py Fri Jun 14 17:49:31 2013
@@ -20,6 +20,7 @@ limitations under the License.
from unittest import TestCase
from PuppetExecutor import PuppetExecutor
+from RepoInstaller import RepoInstaller
from Grep import Grep
from pprint import pformat
import socket, threading, tempfile
@@ -67,6 +68,39 @@ class TestPuppetExecutor(TestCase):
self.assertFalse(puppetInstance.reposInstalled)
os.unlink(tmpdir + os.sep + 'site-' + str(parsedJson["taskId"]) + '.pp')
+ @patch.object(RepoInstaller, 'generate_repo_manifests')
+ @patch.object(PuppetExecutor, 'runPuppetFile')
+ def test_overwrite_repos(self, runPuppetFileMock, generateRepoManifestMock):
+ tmpdir = AmbariConfig().getConfig().get("stack", "installprefix")
+ puppetInstance = PuppetExecutor("/tmp", "/x", "/y", tmpdir, AmbariConfig().getConfig())
+ jsonFile = open('../../main/python/ambari_agent/test.json', 'r')
+ jsonStr = jsonFile.read()
+ parsedJson = json.loads(jsonStr)
+ parsedJson["taskId"] = 77
+ def side_effect(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile):
+ result["exitcode"] = 0
+ runPuppetFileMock.side_effect = side_effect
+
+ #If ambari-agent has been just started and no any commands were executed by
+ # PuppetExecutor.runCommand, then no repo files were updated by
+ # RepoInstaller.generate_repo_manifests
+ self.assertEquals(0, generateRepoManifestMock.call_count)
+ self.assertFalse(puppetInstance.reposInstalled)
+
+ # After executing of the first command, RepoInstaller.generate_repo_manifests
+ # generates a .pp file for updating repo files
+ puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
+ self.assertTrue(puppetInstance.reposInstalled)
+ self.assertEquals(1, generateRepoManifestMock.call_count)
+
+ # After executing of the next commands, repo manifest aren't generated again
+ puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
+ self.assertTrue(puppetInstance.reposInstalled)
+ self.assertEquals(1, generateRepoManifestMock.call_count)
+ puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
+ self.assertTrue(puppetInstance.reposInstalled)
+ self.assertEquals(1, generateRepoManifestMock.call_count)
+
@patch("os.path.exists")
def test_configure_environ(self, osPathExistsMock):
config = AmbariConfig().getConfig()
Modified: incubator/ambari/trunk/ambari-server/sbin/ambari-server
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/sbin/ambari-server?rev=1493187&r1=1493186&r2=1493187&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/sbin/ambari-server (original)
+++ incubator/ambari/trunk/ambari-server/sbin/ambari-server Fri Jun 14 17:49:31 2013
@@ -86,6 +86,10 @@ case "$1" in
echo -e "Upgrading stack of ambari-server"
$PYTHON /usr/sbin/ambari-server.py $@
;;
+ update-metainfo)
+ echo -e "Updating ambari-server meta information"
+ $PYTHON /usr/sbin/ambari-server.py $@
+ ;;
setup)
initdb_res=`/sbin/service postgresql initdb 2>&1 > /dev/null`
initdb_res=`/sbin/service postgresql start 2>&1 > /dev/null`
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java?rev=1493187&r1=1493186&r2=1493187&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java Fri Jun 14 17:49:31 2013
@@ -118,6 +118,7 @@ public class AmbariMetaInfo {
*/
@Inject
public void init() throws Exception {
+ stacksResult = new ArrayList<StackInfo>();
readServerVersion();
getConfigurationInformation(stackRoot);
}
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java?rev=1493187&r1=1493186&r2=1493187&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/api/services/StacksService.java Fri Jun 14 17:49:31 2013
@@ -22,10 +22,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
+import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
@@ -45,7 +42,15 @@ public class StacksService extends BaseS
public Response getStacks(@Context HttpHeaders headers, @Context UriInfo ui) {
return handleRequest(headers, null, ui, Request.Type.GET,
- createStackResource(null));
+ createStackResource(null));
+ }
+
+ @PUT
+ @Produces("text/plain")
+ public Response updateStacks(@Context HttpHeaders headers, @Context UriInfo ui) {
+
+ return handleRequest(headers, null, ui, Request.Type.PUT,
+ createStackResource(null));
}
@GET
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java?rev=1493187&r1=1493186&r2=1493187&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java Fri Jun 14 17:49:31 2013
@@ -423,7 +423,14 @@ public interface AmbariManagementControl
* @throws AmbariException if the resources cannot be read
*/
public Set<StackResponse> getStacks(Set<StackRequest> requests) throws AmbariException;
-
+
+ /**
+ * Update stacks from the files at stackRoot.
+ *
+ * @return a track action response
+ * @throws AmbariException if
+ */
+ public RequestStatusResponse updateStacks() throws AmbariException;
/**
* Get supported stacks versions.
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java?rev=1493187&r1=1493186&r2=1493187&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java Fri Jun 14 17:49:31 2013
@@ -20,16 +20,8 @@ package org.apache.ambari.server.control
import java.io.File;
import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.ClusterNotFoundException;
@@ -4228,6 +4220,19 @@ public class AmbariManagementControllerI
}
@Override
+ public synchronized RequestStatusResponse updateStacks() throws AmbariException {
+
+ try {
+ ambariMetaInfo.init();
+ } catch (Exception e) {
+ throw new AmbariException(
+ "Ambari metainormation can't be read from the stack root directory");
+ }
+
+ return null;
+ }
+
+ @Override
public Set<RepositoryResponse> getRepositories(Set<RepositoryRequest> requests)
throws AmbariException {
Set<RepositoryResponse> response = new HashSet<RepositoryResponse>();
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackResourceProvider.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackResourceProvider.java?rev=1493187&r1=1493186&r2=1493187&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackResourceProvider.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackResourceProvider.java Fri Jun 14 17:49:31 2013
@@ -18,24 +18,15 @@
package org.apache.ambari.server.controller.internal;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.RequestStatusResponse;
import org.apache.ambari.server.controller.StackRequest;
import org.apache.ambari.server.controller.StackResponse;
-import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
-import org.apache.ambari.server.controller.spi.NoSuchResourceException;
-import org.apache.ambari.server.controller.spi.Predicate;
-import org.apache.ambari.server.controller.spi.Request;
-import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.spi.*;
import org.apache.ambari.server.controller.spi.Resource.Type;
-import org.apache.ambari.server.controller.spi.SystemException;
-import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
import org.apache.ambari.server.controller.utilities.PredicateHelper;
import org.apache.ambari.server.controller.utilities.PropertyHelper;
@@ -87,6 +78,25 @@ public class StackResourceProvider exten
return resources;
}
+ @Override
+ public RequestStatus updateResources(Request request, Predicate predicate)
+ throws SystemException, UnsupportedPropertyException,
+ NoSuchResourceException, NoSuchParentResourceException {
+
+ RequestStatusResponse response = modifyResources(
+ new Command<RequestStatusResponse>() {
+
+ @Override
+ public RequestStatusResponse invoke() throws AmbariException {
+ return getManagementController().updateStacks();
+ }
+ });
+
+ notifyUpdate(Type.Stack, request, predicate);
+
+ return getRequestStatus(response);
+ }
+
private StackRequest getRequest(Map<String, Object> properties) {
return new StackRequest((String) properties.get(STACK_NAME_PROPERTY_ID));
}
Modified: incubator/ambari/trunk/ambari-server/src/main/python/ambari-server.py
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/python/ambari-server.py?rev=1493187&r1=1493186&r2=1493187&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/python/ambari-server.py (original)
+++ incubator/ambari/trunk/ambari-server/src/main/python/ambari-server.py Fri Jun 14 17:49:31 2013
@@ -35,6 +35,7 @@ import fileinput
import urllib2
import time
import getpass
+import socket
import datetime
import socket
import tempfile
@@ -51,6 +52,7 @@ STOP_ACTION = "stop"
RESET_ACTION = "reset"
UPGRADE_ACTION = "upgrade"
UPGRADE_STACK_ACTION = "upgradestack"
+UPDATE_METAINFO_ACTION = "update-metainfo"
STATUS_ACTION = "status"
LDAP_SETUP_ACTION = "setupldap"
RESET_MASTER_KEY_ACTION = "resetmasterkey"
@@ -162,6 +164,7 @@ SETUP_DB_CMD = ['su', '-', 'postgres',
'--command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'"']
UPGRADE_STACK_CMD = ['su', 'postgres',
'--command=psql -f {0} -v stack_name="\'{1}\'" -v stack_version="\'{2}\'"']
+UPDATE_METAINFO_CMD = 'curl -X PUT "http://{0}:{1}/api/v1/stacks2" -u "{2}":"{3}"'
PG_ST_CMD = "/sbin/service postgresql status"
PG_INITDB_CMD = "/sbin/service postgresql initdb"
PG_START_CMD = "/sbin/service postgresql start"
@@ -184,6 +187,9 @@ JDBC_PASSWORD_PROPERTY = "server.jdbc.us
JDBC_PASSWORD_FILENAME = "password.dat"
JDBC_RCA_PASSWORD_FILENAME = "rca_password.dat"
+CLIENT_API_PORT_PROPERTY = "client.api.port"
+CLIENT_API_PORT = "8080"
+
PERSISTENCE_TYPE_PROPERTY = "server.persistence.type"
JDBC_DRIVER_PROPERTY = "server.jdbc.driver"
JDBC_URL_PROPERTY = "server.jdbc.url"
@@ -415,6 +421,65 @@ def run_os_command(cmd):
(stdoutdata, stderrdata) = process.communicate()
return process.returncode, stdoutdata, stderrdata
+#
+# Updates metainfo information from stack root. Re-cache information from
+# repoinfo.xml , metainfo.xml files , etc.
+#
+def update_metainfo(args):
+ configure_update_metainfo_args(args)
+
+ hostname = args.hostname
+ port = args.port
+ username = args.username
+ password = args.password
+
+ command = UPDATE_METAINFO_CMD
+ command = command.format(hostname, port, username, password)
+ retcode, outdata, errdata = run_os_command(command)
+
+ if outdata.find("Bad credentials") > 0:
+ print 'Incorrect credential provided. Please try again.'
+
+ if not retcode == 0:
+ print errdata
+ return retcode
+
+def configure_update_metainfo_args(args):
+ conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
+ properties = Properties()
+
+ try:
+ properties.load(open(conf_file))
+ except Exception, e:
+ print 'Could not read ambari config file "%s": %s' % (conf_file, e)
+ return -1
+
+
+ default_username = "admin"
+
+ username_prompt = 'Username [' + default_username + ']: '
+ password_prompt = 'Password: '
+ input_pattern = "^[a-zA-Z_][a-zA-Z0-9_\-]*$"
+
+ hostname = socket.gethostname()
+ port = properties[CLIENT_API_PORT_PROPERTY]
+
+ if not port:
+ port = CLIENT_API_PORT
+
+ input_descr = "Invalid characters in received. Start with _ or alpha "\
+ "followed by alphanumeric or _ or - characters"
+
+ print 'Full authentication is required to access the Ambari API'
+ username = get_validated_string_input(username_prompt, default_username,
+ input_pattern, input_descr, False)
+ password = get_validated_string_input(password_prompt, "", input_pattern,
+ input_descr, True)
+
+ args.hostname = hostname
+ args.port = port
+ args.username = username
+ args.password = password
#
# Checks SELinux
@@ -2543,6 +2608,8 @@ def main():
setup_ldap()
elif action == RESET_MASTER_KEY_ACTION:
reset_master_key()
+ elif action == UPDATE_METAINFO_ACTION:
+ update_metainfo(options)
else:
parser.error("Invalid action")
except FatalException as e:
Modified: incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java?rev=1493187&r1=1493186&r2=1493187&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java (original)
+++ incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java Fri Jun 14 17:49:31 2013
@@ -18,10 +18,6 @@
package org.apache.ambari.server.controller;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -70,6 +66,7 @@ import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.persist.PersistService;
+import static org.junit.Assert.*;
public class AmbariManagementControllerTest {
@@ -92,6 +89,7 @@ public class AmbariManagementControllerT
private static final int OS_CNT = 2;
private static final String NON_EXT_VALUE = "XXX";
+ private static final String INCORRECT_BASE_URL = "http://incorrect.url";
private static final String COMPONENT_NAME = "NAMENODE";
@@ -6480,5 +6478,25 @@ public class AmbariManagementControllerT
State.INSTALLED.toString());
controller.updateServices(Collections.singleton(req), Collections.<String, String>emptyMap(), true, false);
}
+
+ @Test
+ public void testUpdateStacks() throws Exception {
+
+ StackInfo stackInfo = ambariMetaInfo.getStackInfo(STACK_NAME, STACK_VERSION);
+
+ for (RepositoryInfo repositoryInfo: stackInfo.getRepositories()) {
+ assertFalse(INCORRECT_BASE_URL.equals(repositoryInfo.getBaseUrl()));
+ repositoryInfo.setBaseUrl(INCORRECT_BASE_URL);
+ assertTrue(INCORRECT_BASE_URL.equals(repositoryInfo.getBaseUrl()));
+ }
+
+ controller.updateStacks();
+
+ stackInfo = ambariMetaInfo.getStackInfo(STACK_NAME, STACK_VERSION);
+
+ for (RepositoryInfo repositoryInfo: stackInfo.getRepositories()) {
+ assertFalse(INCORRECT_BASE_URL.equals(repositoryInfo.getBaseUrl()));
+ }
+ }
}