You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2013/10/02 19:59:41 UTC
git commit: AMBARI-3430. Add repository url verification. (ncole)
Updated Branches:
refs/heads/trunk 65fbee708 -> f7ab06dfa
AMBARI-3430. Add repository url verification. (ncole)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/f7ab06df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/f7ab06df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/f7ab06df
Branch: refs/heads/trunk
Commit: f7ab06dfa8ddfe138b7ba462ccc62223a2b2186b
Parents: 65fbee7
Author: Nate Cole <nc...@hortonworks.com>
Authored: Wed Oct 2 12:59:36 2013 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Wed Oct 2 13:59:24 2013 -0400
----------------------------------------------------------------------
.../server/configuration/Configuration.java | 19 +++++
.../AmbariManagementControllerImpl.java | 47 +++++++++++-
.../server/controller/RepositoryRequest.java | 15 ++++
.../internal/RepositoryResourceProvider.java | 7 ++
.../src/main/resources/properties.json | 1 +
.../AmbariManagementControllerTest.java | 81 +++++++++++++++++++-
6 files changed, 165 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/f7ab06df/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 3c8600b..1684427 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -264,6 +264,15 @@ public class Configuration {
*/
public static final String RCO_FILE_LOCATION_TEST = "../ambari-common/src/" +
"main/resources/role_command_order.json".replace("/", File.separator);
+
+ /**
+ * Key for repo validation suffixes.
+ */
+ public static final String REPO_SUFFIX_KEY = "repo.validation.suffixes";
+ /**
+ * Default for repo validation suffixes.
+ */
+ private static final String REPO_SUFFIX_DEFAULT = "/repodata/repomd.xml";
private static final Logger LOG = LoggerFactory.getLogger(
Configuration.class);
@@ -837,5 +846,15 @@ public class Configuration {
return value;
}
+
+ /**
+ * @return a string array of suffixes used to validate repo URLs.
+ */
+ public String[] getRepoValidationSuffixes() {
+ String value = properties.getProperty(REPO_SUFFIX_KEY,
+ REPO_SUFFIX_DEFAULT);
+
+ return value.split(",");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/f7ab06df/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index dd37821..1c0f56c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -19,6 +19,7 @@
package org.apache.ambari.server.controller;
import java.io.File;
+import java.io.IOException;
import java.net.InetAddress;
import java.text.MessageFormat;
import java.util.ArrayList;
@@ -54,6 +55,7 @@ import org.apache.ambari.server.actionmanager.StageFactory;
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.controller.internal.URLStreamProvider;
import org.apache.ambari.server.metadata.ActionMetadata;
import org.apache.ambari.server.metadata.RoleCommandOrder;
import org.apache.ambari.server.security.authorization.AuthorizationHelper;
@@ -92,6 +94,7 @@ import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStartEvent
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStopEvent;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostUpgradeEvent;
import org.apache.ambari.server.utils.StageUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger;
@@ -158,7 +161,7 @@ public class AmbariManagementControllerImpl implements
final private static String JDK_RESOURCE_LOCATION =
"/resources/";
- final private static int REPO_URL_CONNECT_TIMEOUT = 2000;
+ final private static int REPO_URL_CONNECT_TIMEOUT = 3000;
final private static int REPO_URL_READ_TIMEOUT = 500;
final private String jdkResourceUrl;
@@ -4450,9 +4453,45 @@ public class AmbariManagementControllerImpl implements
throw new AmbariException("Repo ID must be specified.");
if (null != rr.getBaseUrl()) {
- ambariMetaInfo.updateRepoBaseURL(rr.getStackName(),
- rr.getStackVersion(), rr.getOsType(), rr.getRepoId(),
- rr.getBaseUrl());
+ if (!rr.isVerifyBaseUrl()) {
+ ambariMetaInfo.updateRepoBaseURL(rr.getStackName(),
+ rr.getStackVersion(), rr.getOsType(), rr.getRepoId(),
+ rr.getBaseUrl());
+ } else {
+ URLStreamProvider usp = new URLStreamProvider(REPO_URL_CONNECT_TIMEOUT,
+ REPO_URL_READ_TIMEOUT, null, null, null);
+
+ boolean bFound = false;
+
+ String[] suffixes = configs.getRepoValidationSuffixes();
+ for (int i = 0; i < suffixes.length && !bFound; i++) {
+ String suffix = suffixes[i];
+ String spec = rr.getBaseUrl();
+
+ if (spec.charAt(spec.length()-1) != '/' && suffix.charAt(0) != '/')
+ spec = rr.getBaseUrl() + "/" + suffix;
+ else if (spec.charAt(spec.length()-1) == '/' && suffix.charAt(0) == '/')
+ spec = rr.getBaseUrl() + suffix.substring(1);
+ else
+ spec = rr.getBaseUrl() + suffix;
+
+ try {
+ IOUtils.readLines(usp.readFrom(spec));
+ bFound = true;
+ } catch (IOException ioe) {
+ // failed, but try other suffixes
+ }
+ }
+
+ if (bFound) {
+ ambariMetaInfo.updateRepoBaseURL(rr.getStackName(),
+ rr.getStackVersion(), rr.getOsType(), rr.getRepoId(),
+ rr.getBaseUrl());
+ } else {
+ throw new IllegalArgumentException("Could not access base url '" +
+ rr.getBaseUrl() + "'");
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/f7ab06df/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryRequest.java
index fea86bf..0e1e81f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryRequest.java
@@ -22,6 +22,7 @@ public class RepositoryRequest extends OperatingSystemRequest{
private String repoId;
private String baseUrl;
+ private boolean verify = true;
public RepositoryRequest(String stackName, String stackVersion, String osType, String repoId) {
super(stackName, stackVersion, osType);
@@ -54,6 +55,20 @@ public class RepositoryRequest extends OperatingSystemRequest{
baseUrl = url;
}
+ /**
+ * @return <code>true</code> if the base url should be verified. Default is <code>true</code>.
+ */
+ public boolean isVerifyBaseUrl() {
+ return verify;
+ }
+
+ /**
+ * @param verifyUrl <code>true</code> to verify the base url
+ */
+ public void setVerifyBaseUrl(boolean verifyUrl) {
+ verify = verifyUrl;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/f7ab06df/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java
index 7cea0af..99139a4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryResourceProvider.java
@@ -72,6 +72,9 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
public static final String REPOSITORY_DEFAULT_BASE_URL_PROPERTY_ID = PropertyHelper
.getPropertyId("Repositories", "default_base_url");
+ public static final String REPOSITORY_VERIFY_BASE_URL_PROPERTY_ID = PropertyHelper
+ .getPropertyId("Repositories", "verify_base_url");
+
private static Set<String> pkPropertyIds = new HashSet<String>(
Arrays.asList(new String[] { STACK_NAME_PROPERTY_ID,
@@ -184,6 +187,10 @@ public class RepositoryResourceProvider extends AbstractControllerResourceProvid
if (properties.containsKey(REPOSITORY_BASE_URL_PROPERTY_ID)) {
rr.setBaseUrl((String) properties.get(REPOSITORY_BASE_URL_PROPERTY_ID));
+
+ if (properties.containsKey(REPOSITORY_VERIFY_BASE_URL_PROPERTY_ID)) {
+ rr.setVerifyBaseUrl("true".equalsIgnoreCase(properties.get(REPOSITORY_VERIFY_BASE_URL_PROPERTY_ID).toString()));
+ }
}
return rr;
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/f7ab06df/ambari-server/src/main/resources/properties.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json
index 9148c1a..9f488bc 100644
--- a/ambari-server/src/main/resources/properties.json
+++ b/ambari-server/src/main/resources/properties.json
@@ -138,6 +138,7 @@
"Repositories/repo_name",
"Repositories/mirrors_list",
"Repositories/default_base_url",
+ "Repositories/verify_base_url",
"_"
],
"StackService":[
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/f7ab06df/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 652d7fa..a18bcc6 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -32,6 +32,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import javax.persistence.EntityManager;
@@ -140,13 +141,17 @@ public class AmbariManagementControllerTest {
private AmbariMetaInfo ambariMetaInfo;
private Users users;
private EntityManager entityManager;
+ private Properties backingProperties;
+ private Configuration configuration;
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Before
public void setup() throws Exception {
- injector = Guice.createInjector(new InMemoryDefaultTestModule());
+ InMemoryDefaultTestModule module = new InMemoryDefaultTestModule();
+ backingProperties = module.getProperties();
+ injector = Guice.createInjector(module);
injector.getInstance(GuiceJpaInitializer.class);
entityManager = injector.getInstance(EntityManager.class);
clusters = injector.getInstance(Clusters.class);
@@ -160,6 +165,7 @@ public class AmbariManagementControllerTest {
ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class);
ambariMetaInfo.init();
users = injector.getInstance(Users.class);
+ configuration = injector.getInstance(Configuration.class);
}
@After
@@ -6871,6 +6877,79 @@ public class AmbariManagementControllerTest {
assertEquals(original, repo.getBaseUrl());
assertEquals(original, repo.getDefaultBaseUrl());
}
+
+ @Test
+ public void testUpdateRepoUrlController() throws Exception {
+ RepositoryInfo repo = ambariMetaInfo.getRepository(STACK_NAME, STACK_VERSION, OS_TYPE, REPO_ID);
+
+ RepositoryRequest request = new RepositoryRequest(STACK_NAME, STACK_VERSION, OS_TYPE, REPO_ID);
+ request.setBaseUrl("http://hortonworks.com");
+
+ Set<RepositoryRequest> requests = new HashSet<RepositoryRequest>();
+ requests.add(request);
+
+ // test bad url
+ try {
+ controller.updateRespositories(requests);
+ fail ("Expected exception on invalid url");
+ } catch (Exception e) {
+ }
+
+ // test bad url, but allow to set anyway
+ request.setVerifyBaseUrl(false);
+ controller.updateRespositories(requests);
+ Assert.assertEquals(request.getBaseUrl(), repo.getBaseUrl());
+
+ // reset repo
+ requests.clear();
+ request = new RepositoryRequest(STACK_NAME, STACK_VERSION, OS_TYPE, REPO_ID);
+ request.setBaseUrl(repo.getDefaultBaseUrl());
+ requests.add(request);
+ controller.updateRespositories(requests);
+ Assert.assertEquals(repo.getBaseUrl(), repo.getDefaultBaseUrl());
+
+ String baseUrl = repo.getDefaultBaseUrl();
+ if (!baseUrl.endsWith("/"))
+ baseUrl += "/";
+
+ // variation #1: url with trailing slash, suffix preceding slash
+ backingProperties.setProperty(Configuration.REPO_SUFFIX_KEY, "/repodata/repomd.xml");
+ Assert.assertTrue(baseUrl.endsWith("/") && configuration.getRepoValidationSuffixes()[0].startsWith("/"));
+ request.setBaseUrl(baseUrl);
+ controller.updateRespositories(requests);
+ Assert.assertEquals(baseUrl, repo.getBaseUrl());
+
+ // variation #2: url with trailing slash, suffix no preceding slash
+ backingProperties.setProperty(Configuration.REPO_SUFFIX_KEY, "repodata/repomd.xml");
+ Assert.assertTrue(baseUrl.endsWith("/") && !configuration.getRepoValidationSuffixes()[0].startsWith("/"));
+ request.setBaseUrl(baseUrl);
+ controller.updateRespositories(requests);
+ Assert.assertEquals(baseUrl, repo.getBaseUrl());
+
+ baseUrl = baseUrl.substring(0, baseUrl.length()-1);
+ // variation #3: url with no trailing slash, suffix no prededing slash
+ Assert.assertTrue(!baseUrl.endsWith("/") && !configuration.getRepoValidationSuffixes()[0].startsWith("/"));
+ request.setBaseUrl(baseUrl);
+ controller.updateRespositories(requests);
+ Assert.assertEquals(baseUrl, repo.getBaseUrl());
+
+ // variation #4: url with no trailing slash, suffix preceding slash
+ backingProperties.setProperty(Configuration.REPO_SUFFIX_KEY, "/repodata/repomd.xml");
+ Assert.assertTrue(!baseUrl.endsWith("/") && configuration.getRepoValidationSuffixes()[0].startsWith("/"));
+ request.setBaseUrl(baseUrl);
+ controller.updateRespositories(requests);
+ Assert.assertEquals(baseUrl, repo.getBaseUrl());
+
+ // variation #5: multiple suffix tests
+ backingProperties.setProperty(Configuration.REPO_SUFFIX_KEY, "/foo/bar.xml,/repodata/repomd.xml");
+ Assert.assertTrue(configuration.getRepoValidationSuffixes().length > 1);
+ request.setBaseUrl(baseUrl);
+ controller.updateRespositories(requests);
+ Assert.assertEquals(baseUrl, repo.getBaseUrl());
+
+ }
+
+
@Test
public void testDeleteHostComponentInVariousStates() throws Exception {