You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ma...@apache.org on 2016/09/15 13:21:02 UTC
[2/2] ambari git commit: AMBARI-15538. Support service-specific repo
for add-on services (Balazs bence Sari via magyari_sandor)
AMBARI-15538. Support service-specific repo for add-on services (Balazs bence Sari via magyari_sandor)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7961cd11
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7961cd11
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7961cd11
Branch: refs/heads/branch-2.5
Commit: 7961cd114566d035ee844098865195667d1cb4f5
Parents: 6a947a9
Author: Balazs Bence Sari <bs...@hortonworks.com>
Authored: Thu Sep 15 15:08:16 2016 +0200
Committer: Sandor Magyari <sm...@hortonworks.com>
Committed: Thu Sep 15 15:20:27 2016 +0200
----------------------------------------------------------------------
.../checks/DatabaseConsistencyCheckHelper.java | 1 +
.../AmbariManagementControllerImpl.java | 13 +-
.../ambari/server/controller/AmbariServer.java | 2 +
.../VersionDefinitionResourceProvider.java | 18 +-
.../apache/ambari/server/stack/RepoUtil.java | 208 +++++++++++++++++++
.../ambari/server/stack/ServiceModule.java | 7 +
.../ambari/server/stack/StackDirectory.java | 23 +-
.../apache/ambari/server/stack/StackModule.java | 135 ++++++++++--
.../server/stack/StackServiceDirectory.java | 70 ++++++-
.../stack/UpdateActiveRepoVersionOnStartup.java | 118 +++++++++++
.../ambari/server/state/RepositoryInfo.java | 57 +++++
.../apache/ambari/server/state/StackInfo.java | 13 +-
.../stack/upgrade/RepositoryVersionHelper.java | 28 ++-
.../src/main/resources/version_definition.xsd | 24 +--
.../ambari/server/stack/RepoUtilTest.java | 166 +++++++++++++++
.../stack/StackManagerCommonServicesTest.java | 20 ++
.../ambari/server/stack/StackModuleTest.java | 188 +++++++++++++++++
.../UpdateActiveRepoVersionOnStartupTest.java | 143 +++++++++++++
.../ADDON/1.0/configuration/addon-env.xml | 35 ++++
.../common-services/ADDON/1.0/metainfo.xml | 35 ++++
...veRepoVersionOnStartupTest_initialRepos.json | 32 +++
.../HDP/0.2/services/ADDON/metainfo.xml | 28 +++
.../HDP/0.2/services/ADDON/repos/repoinfo.xml | 26 +++
.../8.0.0/configuration/microsoft-r-env.xml | 35 ++++
.../8.0.0/package/scripts/microsoft_r.py | 22 +-
.../MICROSOFT_R/8.0.0/repos/repoinfo.xml | 33 +++
26 files changed, 1400 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
index f302b8b..2d91eca 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
@@ -545,6 +545,7 @@ public class DatabaseConsistencyCheckHelper {
String stackVersion = stackInfo.get(stackName);
Map<String, ServiceInfo> serviceInfoMap = ambariMetaInfo.getServices(stackName, stackVersion);
for (String serviceName : serviceNames) {
+ LOG.info("Processing {}-{} / {}", stackName, stackVersion, serviceName);
ServiceInfo serviceInfo = serviceInfoMap.get(serviceName);
if (serviceInfo != null) {
Set<String> configTypes = serviceInfo.getConfigTypeAttributes().keySet();
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 3acf490..a35b0e9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -140,6 +140,7 @@ import org.apache.ambari.server.security.ldap.LdapSyncDto;
import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException;
import org.apache.ambari.server.serveraction.kerberos.KerberosOperationException;
import org.apache.ambari.server.stack.ExtensionHelper;
+import org.apache.ambari.server.stack.RepoUtil;
import org.apache.ambari.server.stageplanner.RoleGraph;
import org.apache.ambari.server.stageplanner.RoleGraphFactory;
import org.apache.ambari.server.state.Cluster;
@@ -206,6 +207,7 @@ import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Singleton;
import com.google.inject.persist.Transactional;
+import com.google.common.collect.ListMultimap;
@Singleton
public class AmbariManagementControllerImpl implements AmbariManagementController {
@@ -2286,7 +2288,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
hostParams.put(PACKAGE_LIST, packageList);
Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs();
-
+
Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredConfigs);
String userList = gson.toJson(userSet);
hostParams.put(USER_LIST, userList);
@@ -2326,7 +2328,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
execCmd.setRoleParams(roleParams);
-
+
execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()));
if ((execCmd != null) && (execCmd.getConfigurationTags().containsKey("cluster-env"))) {
@@ -4082,7 +4084,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
StackId stackId = new StackId(xml.release.stackId);
+ ListMultimap<String, RepositoryInfo> stackRepositoriesByOs = ambariMetaInfo.getStackManager().getStack(stackName, stackVersion).getRepositoriesByOs();
for (RepositoryXml.Os os : xml.repositoryInfo.getOses()) {
+
for (RepositoryXml.Repo repo : os.getRepos()) {
RepositoryResponse resp = new RepositoryResponse(repo.getBaseUrl(), os.getFamily(),
repo.getRepoId(), repo.getRepoName(), repo.getMirrorsList(),
@@ -4096,6 +4100,11 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
}
+ // Add service repos to the response. (These are not contained by the VDF but are present in the stack model)
+ List<RepositoryInfo> serviceRepos =
+ RepoUtil.getServiceRepos(xml.repositoryInfo.getRepositories(), stackRepositoriesByOs);
+ responses.addAll(RepoUtil.asResponses(serviceRepos, versionDefinitionId, stackName, stackVersion));
+
} else {
if (repoId == null) {
List<RepositoryInfo> repositories = ambariMetaInfo.getRepositories(stackName, stackVersion, osType);
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index 097f01c..89cdb93 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -108,6 +108,7 @@ import org.apache.ambari.server.security.ldap.AmbariLdapDataPopulator;
import org.apache.ambari.server.security.unsecured.rest.CertificateDownload;
import org.apache.ambari.server.security.unsecured.rest.CertificateSign;
import org.apache.ambari.server.security.unsecured.rest.ConnectionInfo;
+import org.apache.ambari.server.stack.UpdateActiveRepoVersionOnStartup;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.topology.AmbariContext;
import org.apache.ambari.server.topology.BlueprintFactory;
@@ -918,6 +919,7 @@ public class AmbariServer {
injector.getInstance(GuiceJpaInitializer.class);
DatabaseConsistencyCheckHelper.checkDBVersionCompatible();
server = injector.getInstance(AmbariServer.class);
+ injector.getInstance(UpdateActiveRepoVersionOnStartup.class).process();
CertificateManager certMan = injector.getInstance(CertificateManager.class);
certMan.initRootCert();
KerberosChecker.checkJaasConfiguration();
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java
index 02fc2ec..629f3cd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java
@@ -54,6 +54,8 @@ import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.orm.entities.StackEntity;
import org.apache.ambari.server.security.authorization.ResourceType;
import org.apache.ambari.server.security.authorization.RoleAuthorization;
+import org.apache.ambari.server.stack.RepoUtil;
+import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.RepositoryType;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.StackInfo;
@@ -72,6 +74,7 @@ import org.codehaus.jackson.node.ObjectNode;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
import com.google.inject.Provider;
+import com.google.common.collect.ListMultimap;
/**
* The {@link VersionDefinitionResourceProvider} class deals with managing Version Definition
@@ -237,7 +240,7 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc
try {
holder.xmlString = xml.toXml();
} catch (Exception e) {
- throw new AmbariException(String.format("The available repository %s does not serialize", definitionName));
+ throw new AmbariException(String.format("The available repository %s does not serialize", definitionName), e);
}
} else {
@@ -559,8 +562,16 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc
StackEntity stackEntity = s_stackDAO.find(stackId.getStackName(), stackId.getStackVersion());
entity.setStack(stackEntity);
- entity.setOperatingSystems(s_repoVersionHelper.get().serializeOperatingSystems(
- holder.xml.repositoryInfo.getRepositories()));
+
+ List<RepositoryInfo> repos = holder.xml.repositoryInfo.getRepositories();
+
+ // Add service repositories (these are not contained by the VDF but are there in the stack model)
+ ListMultimap<String, RepositoryInfo> stackReposByOs =
+ s_metaInfo.get().getStack(stackId.getStackName(), stackId.getStackVersion()).getRepositoriesByOs();
+ repos.addAll(RepoUtil.getServiceRepos(repos, stackReposByOs));
+
+ entity.setOperatingSystems(s_repoVersionHelper.get().serializeOperatingSystems(repos));
+
entity.setVersion(holder.xml.release.getFullVersion());
entity.setDisplayName(stackId, holder.xml.release);
entity.setType(holder.xml.release.repositoryType);
@@ -723,7 +734,6 @@ public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourc
entity.getStackName());
repoElement.put(PropertyHelper.getPropertyName(RepositoryResourceProvider.REPOSITORY_STACK_VERSION_PROPERTY_ID),
entity.getStackVersion());
-
repoBase.put(PropertyHelper.getPropertyCategory(RepositoryResourceProvider.REPOSITORY_BASE_URL_PROPERTY_ID),
repoElement);
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java
new file mode 100644
index 0000000..07b845a
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java
@@ -0,0 +1,208 @@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.stack;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import javax.annotation.Nullable;
+import javax.xml.bind.JAXBException;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.RepositoryResponse;
+import org.apache.ambari.server.orm.entities.OperatingSystemEntity;
+import org.apache.ambari.server.orm.entities.RepositoryEntity;
+import org.apache.ambari.server.state.RepositoryInfo;
+import org.apache.ambari.server.state.stack.RepositoryXml;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.Sets;
+
+/**
+ * Utility functions for repository replated tasks.
+ */
+public class RepoUtil {
+
+ /**
+ * logger instance
+ */
+ private final static Logger LOG = LoggerFactory.getLogger(RepoUtil.class);
+
+
+ /**
+ * repository directory name
+ */
+ final static String REPOSITORY_FOLDER_NAME = "repos";
+
+ /**
+ * repository file name
+ */
+ final static String REPOSITORY_FILE_NAME = "repoinfo.xml";
+
+ private static final Function<RepositoryEntity, String> REPO_ENTITY_TO_NAME = new Function<RepositoryEntity, String>() {
+ @Override public String apply(@Nullable RepositoryEntity input) { return input.getName(); }
+ };
+
+
+ /**
+ * Parses the repository file for a stack/service if exists.
+ *
+ * @param directory stack/service base directory
+ * @param subDirs stack/service directory sub directories
+ * @param unmarshaller {@link ModuleFileUnmarshaller}, needed to parse repo XML
+ * @throws AmbariException if unable to parse the repository file
+ * @return The directory containing the repo file and the parsed repo file (if exists)
+ */
+ public static RepositoryFolderAndXml parseRepoFile(File directory,
+ Collection<String> subDirs,
+ ModuleFileUnmarshaller unmarshaller) {
+ File repositoryFile = null;
+ String repoDir = null;
+ RepositoryXml repoFile = null;
+
+ if (subDirs.contains(REPOSITORY_FOLDER_NAME)) {
+ repoDir = directory.getAbsolutePath() + File.separator + REPOSITORY_FOLDER_NAME;
+ repositoryFile = new File(directory.getPath()+ File.separator +
+ REPOSITORY_FOLDER_NAME + File.separator + REPOSITORY_FILE_NAME);
+
+ if (repositoryFile.exists()) {
+ try {
+ repoFile = unmarshaller.unmarshal(RepositoryXml.class, repositoryFile);
+ } catch (JAXBException e) {
+ repoFile = new RepositoryXml();
+ repoFile.setValid(false);
+ String msg = "Unable to parse repo file at location: " +
+ repositoryFile.getAbsolutePath();
+ repoFile.addError(msg);
+ LOG.warn(msg);
+ }
+ }
+ }
+
+ return new RepositoryFolderAndXml(Optional.fromNullable(repoDir), Optional.fromNullable(repoFile));
+ }
+
+ /**
+ * Checks the passed {@code operatingSystems} parameter if it contains all repositories from the stack model. If a
+ * repository is present in the stack model but missing in the operating system entity list, it is considered a
+ * service repository and will be added.
+ * @param operatingSystems - A list of OperatingSystemEntity objects extracted from a RepositoryVersionEntity
+ * @param stackReposByOs - Stack repositories loaded from the disk (including service repositories), grouped by os.
+ */
+ public static void addServiceReposToOperatingSystemEntities(List<OperatingSystemEntity> operatingSystems,
+ ListMultimap<String, RepositoryInfo> stackReposByOs) {
+ Set<String> addedRepos = new HashSet<>();
+ for (OperatingSystemEntity os : operatingSystems) {
+ List<RepositoryInfo> serviceReposForOs = stackReposByOs.get(os.getOsType());
+ ImmutableSet<String> repoNames = ImmutableSet.copyOf(Lists.transform(os.getRepositories(), REPO_ENTITY_TO_NAME));
+ for (RepositoryInfo repoInfo : serviceReposForOs)
+ if (!repoNames.contains(repoInfo.getRepoName())) {
+ os.getRepositories().add(toRepositoryEntity(repoInfo));
+ addedRepos.add(String.format("%s (%s)", repoInfo.getRepoId(), os.getOsType()));
+ }
+ }
+ LOG.info("Added {} service repos: {}", addedRepos.size(),Iterables.toString(addedRepos));
+ }
+
+ /**
+ * Given a list of VDF repositorie and stack repositories (grouped by os) returns the service repositories.
+ * A repository is considered a service repo if present in the stack model but missing in the VDF (check is performed
+ * by repository name, per operating system).
+ * @param vdfRepos the repositories coming from a version definition
+ * @param stackReposByOs the repositories in the stack model (loaded from disks)
+ * @return A list of service repositories
+ */
+ public static List<RepositoryInfo> getServiceRepos(List<RepositoryInfo> vdfRepos,
+ ListMultimap<String, RepositoryInfo> stackReposByOs) {
+ Set<String> serviceRepoIds = new HashSet<>();
+ List<RepositoryInfo> serviceRepos = new ArrayList<>();
+ ListMultimap<String, RepositoryInfo> vdfReposByOs = Multimaps.index(vdfRepos, RepositoryInfo.GET_OSTYPE_FUNCTION);
+ for(String os: vdfReposByOs.keySet()) {
+ Set<String> vdfRepoNames = Sets.newHashSet(
+ Lists.transform(vdfReposByOs.get(os), RepositoryInfo.GET_REPO_NAME_FUNCTION));
+ for (RepositoryInfo repo: stackReposByOs.get(os)) {
+ if (!vdfRepoNames.contains(repo.getRepoName())) {
+ serviceRepos.add(repo);
+ serviceRepoIds.add(repo.getRepoId());
+ }
+ }
+ }
+ LOG.info("Found {} service repos: {}", serviceRepoIds.size(),Iterables.toString(serviceRepoIds));
+ return serviceRepos;
+ }
+
+ /**
+ * Convert a list of {@link RepositoryInfo} objects to a lost of {@link RepositoryResponse} objects
+ * @param repositoryInfos the list of repository infos
+ * @param versionDefinitionId the version definition id
+ * @param stackName the stack name
+ * @param stackVersion the stack version
+ * @return a list of repository responses
+ */
+ public static List<RepositoryResponse> asResponses(List<RepositoryInfo> repositoryInfos,
+ @Nullable String versionDefinitionId,
+ @Nullable String stackName,
+ @Nullable String stackVersion) {
+ List<RepositoryResponse> responses = new ArrayList<>(repositoryInfos.size());
+ for (RepositoryInfo repoInfo: repositoryInfos) {
+ RepositoryResponse response = repoInfo.convertToResponse();
+ response.setVersionDefinitionId(versionDefinitionId);
+ response.setStackName(stackName);
+ response.setStackVersion(stackVersion);
+ responses.add(response);
+ }
+ return responses;
+ }
+
+ private static RepositoryEntity toRepositoryEntity(RepositoryInfo repoInfo) {
+ RepositoryEntity re = new RepositoryEntity();
+ re.setBaseUrl(repoInfo.getBaseUrl());
+ re.setName(repoInfo.getRepoName());
+ re.setRepositoryId(repoInfo.getRepoId());
+ return re;
+ }
+
+}
+
+/**
+ * Value class for a pair of repository folder and parsed repository XML.
+ */
+class RepositoryFolderAndXml {
+ final Optional<String> repoDir;
+ final Optional<RepositoryXml> repoXml;
+
+ /**
+ * @param repoDir Path to the repository directory (optional)
+ * @param repoXml Parsed repository XML (optional)
+ */
+ public RepositoryFolderAndXml(Optional<String> repoDir, Optional<RepositoryXml> repoXml) {
+ this.repoDir = repoDir;
+ this.repoXml = repoXml;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
index bc94104..a77a22f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
@@ -621,6 +621,13 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem
return errorSet;
}
+ /**
+ * @return The service's directory
+ */
+ public ServiceDirectory getServiceDirectory() {
+ return serviceDirectory;
+ }
+
@Override
public void addErrors(Collection<String> errors) {
this.errorSet.addAll(errors);
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java
index bfba021..c2c8a9e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java
@@ -321,26 +321,9 @@ public class StackDirectory extends StackDefinitionDirectory {
* @throws AmbariException if unable to parse the repository file
*/
private void parseRepoFile(Collection<String> subDirs) throws AmbariException {
- File repositoryFile;
-
- if (subDirs.contains(REPOSITORY_FOLDER_NAME)) {
- repoDir = getAbsolutePath() + File.separator + REPOSITORY_FOLDER_NAME;
- repositoryFile = new File(getPath()+ File.separator +
- REPOSITORY_FOLDER_NAME + File.separator + REPOSITORY_FILE_NAME);
-
- if (repositoryFile.exists()) {
- try {
- repoFile = unmarshaller.unmarshal(RepositoryXml.class, repositoryFile);
- } catch (JAXBException e) {
- repoFile = new RepositoryXml();
- repoFile.setValid(false);
- String msg = "Unable to parse repo file at location: " +
- repositoryFile.getAbsolutePath();
- repoFile.addError(msg);
- LOG.warn(msg);
- }
- }
- }
+ RepositoryFolderAndXml repoDirAndXml = RepoUtil.parseRepoFile(directory, subDirs, unmarshaller);
+ repoDir = repoDirAndXml.repoDir.orNull();
+ repoFile = repoDirAndXml.repoXml.orNull();
if (repoFile == null || !repoFile.isValid()) {
LOG.warn("No repository information defined for "
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
index 0606f2a..bb8d740 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -55,6 +56,13 @@ import org.apache.ambari.server.state.stack.upgrade.ClusterGrouping.ExecuteStage
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Function;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimaps;
+
/**
* Stack module which provides all functionality related to parsing and fully
* resolving stacks from the stack definition.
@@ -290,7 +298,6 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
*
* @param allStacks all stacks in stack definition
* @param commonServices all common services specified in the stack definition
- * @param parentVersion version of the stacks parent
*
* @throws AmbariException if an exception occurs merging with the parent
*/
@@ -1021,38 +1028,130 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
* @throws AmbariException if unable to fully process the stack repositories
*/
private void processRepositories() throws AmbariException {
-
+ List<RepositoryInfo> stackRepos = Collections.emptyList();
RepositoryXml rxml = stackDirectory.getRepoFile();
- if (rxml == null) {
- return;
- }
- stackInfo.setRepositoryXml(rxml);
+ if (null != rxml) {
+ stackInfo.setRepositoryXml(rxml);
- LOG.debug("Adding repositories to stack" +
- ", stackName=" + stackInfo.getName() +
- ", stackVersion=" + stackInfo.getVersion() +
- ", repoFolder=" + stackDirectory.getRepoDir());
+ LOG.debug("Adding repositories to stack" +
+ ", stackName=" + stackInfo.getName() +
+ ", stackVersion=" + stackInfo.getVersion() +
+ ", repoFolder=" + stackDirectory.getRepoDir());
- List<RepositoryInfo> repos = rxml.getRepositories();
+ stackRepos = rxml.getRepositories();
- for (RepositoryInfo ri : repos) {
- processRepository(ri);
+ for (RepositoryInfo ri : stackRepos) {
+ processRepository(ri);
+ }
+
+ stackInfo.getRepositories().addAll(stackRepos);
}
- stackInfo.getRepositories().addAll(repos);
+ LOG.debug("Process service custom repositories");
+ Set<RepositoryInfo> serviceRepos = getUniqueServiceRepos(stackRepos);
+ stackInfo.getRepositories().addAll(serviceRepos);
- if (null != rxml.getLatestURI() && repos.size() > 0) {
+ if (null != rxml && null != rxml.getLatestURI() && stackRepos.size() > 0) {
stackContext.registerRepoUpdateTask(rxml.getLatestURI(), this);
}
}
/**
+ * Gets the service repos with duplicates filtered out. A service repo is considered duplicate if:
+ * <ul>
+ * <li>It has the same name as a stack repo</li>
+ * <li>It has the same id as another service repo</li>
+ * </ul>
+ * Duplicate repo url's only results in warnings in the log. Duplicates are checked per os type, so e.g. the same repo
+ * can exsist for centos5 and centos6.
+ * @param stackRepos the list of stack repositories
+ * @return the service repos with duplicates filtered out.
+ */
+ private Set<RepositoryInfo> getUniqueServiceRepos(List<RepositoryInfo> stackRepos) {
+ List<RepositoryInfo> serviceRepos = getAllServiceRepos();
+ ImmutableListMultimap<String, RepositoryInfo> serviceReposByOsType = Multimaps.index(serviceRepos, RepositoryInfo.GET_OSTYPE_FUNCTION);
+ ImmutableListMultimap<String, RepositoryInfo> stackReposByOsType = Multimaps.index(stackRepos, RepositoryInfo.GET_OSTYPE_FUNCTION);
+
+ Set<RepositoryInfo> uniqueServiceRepos = new HashSet<>();
+
+ // Uniqueness is checked for each os type
+ for (String osType: serviceReposByOsType.keySet()) {
+ List<RepositoryInfo> stackReposForOsType = stackReposByOsType.containsKey(osType) ? stackReposByOsType.get(osType) : Collections.<RepositoryInfo>emptyList();
+ List<RepositoryInfo> serviceReposForOsType = serviceReposByOsType.get(osType);
+ Set<String> stackRepoNames = ImmutableSet.copyOf(Lists.transform(stackReposForOsType, RepositoryInfo.GET_REPO_NAME_FUNCTION));
+ Set<String> stackRepoUrls = ImmutableSet.copyOf(Lists.transform(stackReposForOsType, RepositoryInfo.SAFE_GET_BASE_URL_FUNCTION));
+ Set<String> duplicateServiceRepoNames = findDuplicates(serviceReposForOsType, RepositoryInfo.GET_REPO_NAME_FUNCTION);
+ Set<String> duplicateServiceRepoUrls = findDuplicates(serviceReposForOsType, RepositoryInfo.SAFE_GET_BASE_URL_FUNCTION);
+
+ for (RepositoryInfo repo: serviceReposForOsType) {
+ // These cases only generate warnings
+ if (stackRepoUrls.contains(repo.getBaseUrl())) {
+ LOG.warn("Service repo has a base url that is identical to that of a stack repo: {}", repo);
+ }
+ else if (duplicateServiceRepoUrls.contains(repo.getBaseUrl())) {
+ LOG.warn("Service repo has a base url that is identical to that of another service repo: {}", repo);
+ }
+ // These cases cause the repo to be disregarded
+ if (stackRepoNames.contains(repo.getRepoName())) {
+ LOG.warn("Discarding service repository with the same name as one of the stack repos: {}", repo);
+ }
+ else if (duplicateServiceRepoNames.contains(repo.getRepoName())) {
+ LOG.warn("Discarding service repository with duplicate name and different content: {}", repo);
+ }
+ else {
+ uniqueServiceRepos.add(repo);
+ }
+ }
+ }
+ return uniqueServiceRepos;
+ }
+
+ /**
+ * Finds duplicate repository infos. Duplicateness is checked on the property specified in the keyExtractor.
+ * Items that are equal don't count as duplicate, only differing items with the same key
+ * @param input the input list
+ * @param keyExtractor a function to that returns the property to be checked
+ * @return a set containing the keys of duplicates
+ */
+ private static Set<String> findDuplicates(List<RepositoryInfo> input, Function<RepositoryInfo, String> keyExtractor) {
+ ListMultimap<String, RepositoryInfo> itemsByKey = Multimaps.index(input, keyExtractor);
+ Set<String> duplicates = new HashSet<>();
+ for (Map.Entry<String, Collection<RepositoryInfo>> entry: itemsByKey.asMap().entrySet()) {
+ if (entry.getValue().size() > 1) {
+ Set<RepositoryInfo> differingItems = new HashSet<>();
+ differingItems.addAll(entry.getValue());
+ if (differingItems.size() > 1) {
+ duplicates.add(entry.getKey());
+ }
+ }
+ }
+ return duplicates;
+ }
+
+ /**
+ * Returns all service repositories for a given stack
+ * @return a list of service repo definitions
+ */
+ private List<RepositoryInfo> getAllServiceRepos() {
+ List<RepositoryInfo> repos = new ArrayList<>();
+ for (ServiceModule sm: serviceModules.values()) {
+ ServiceDirectory sd = sm.getServiceDirectory();
+ if (sd instanceof StackServiceDirectory) {
+ StackServiceDirectory ssd = (StackServiceDirectory) sd;
+ RepositoryXml serviceRepoXml = ssd.getRepoFile();
+ if (null != serviceRepoXml) {
+ repos.addAll(serviceRepoXml.getRepositories());
+ }
+ }
+ }
+ return repos;
+ }
+
+ /**
* Process a repository associated with the stack.
*
- * @param osFamily OS family
- * @param osType OS type
- * @param r repo
+ * @param ri The RespositoryInfo to process
*/
private RepositoryInfo processRepository(RepositoryInfo ri) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
index 7bcd08b..a8b4632 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
@@ -18,19 +18,35 @@
package org.apache.ambari.server.stack;
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collection;
+import javax.annotation.Nullable;
+
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.state.stack.RepositoryXml;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-
-
/**
* Encapsulates IO operations on a stack service directory.
*/
public class StackServiceDirectory extends ServiceDirectory {
/**
+ * repository file
+ */
+ @Nullable
+ private RepositoryXml repoFile;
+
+ /**
+ * repository directory
+ */
+ @Nullable
+ private String repoDir;
+
+
+ /**
* logger instance
*/
private static final Logger LOG = LoggerFactory.getLogger(StackServiceDirectory.class);
@@ -45,6 +61,28 @@ public class StackServiceDirectory extends ServiceDirectory {
super(servicePath);
}
+
+ /**
+ * Obtain the repository xml file if exists or null
+ *
+ * @return the repository xml file if exists or null
+ */
+ @Nullable
+ public RepositoryXml getRepoFile() {
+ return repoFile;
+ }
+
+ /**
+ * Obtain the repository directory if exists or null
+ *
+ * @return the repository directory if exists or null
+ */
+ @Nullable
+ public String getRepoDir() {
+ return repoDir;
+ }
+
+
@Override
/**
* Obtain the advisor name.
@@ -65,6 +103,30 @@ public class StackServiceDirectory extends ServiceDirectory {
return stackName + versionString + serviceName + "ServiceAdvisor";
}
+ /**
+ * Parse the repository file.
+ *
+ * @param subDirs service directory sub directories
+ */
+ private void parseRepoFile(Collection<String> subDirs) {
+ RepositoryFolderAndXml repoDirAndXml = RepoUtil.parseRepoFile(directory, subDirs, unmarshaller);
+ repoDir = repoDirAndXml.repoDir.orNull();
+ repoFile = repoDirAndXml.repoXml.orNull();
+
+ if (repoFile == null || !repoFile.isValid()) {
+ LOG.info("No repository information defined for "
+ + ", serviceName=" + getName()
+ + ", repoFolder=" + getPath() + File.separator + RepoUtil.REPOSITORY_FOLDER_NAME);
+ }
+ }
+
+ @Override
+ protected void parsePath() throws AmbariException {
+ super.parsePath();
+ Collection<String> subDirs = Arrays.asList(directory.list());
+ parseRepoFile(subDirs);
+ }
+
@Override
/**
* Calculate the stack service directories.
@@ -116,4 +178,6 @@ public class StackServiceDirectory extends ServiceDirectory {
absUpgradesDir, serviceDir.getName(), stackId);
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartup.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartup.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartup.java
new file mode 100644
index 0000000..1413c66
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartup.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.stack;
+
+import java.util.List;
+import javax.annotation.Nullable;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.orm.dao.ClusterDAO;
+import org.apache.ambari.server.orm.dao.ClusterVersionDAO;
+import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
+import org.apache.ambari.server.orm.entities.ClusterEntity;
+import org.apache.ambari.server.orm.entities.ClusterVersionEntity;
+import org.apache.ambari.server.orm.entities.OperatingSystemEntity;
+import org.apache.ambari.server.orm.entities.RepositoryEntity;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
+import org.apache.ambari.server.state.RepositoryInfo;
+import org.apache.ambari.server.state.StackInfo;
+import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ListMultimap;
+import com.google.inject.Inject;
+import com.google.inject.persist.Transactional;
+
+
+/**
+ * This class should be instantiated on server startup and its {@link #process()} method invoked.
+ * The class is part of management pack support. Management packs can contain services which define
+ * their own (yum/apt/ect) repositories. If a management pack is installed on an Ambari with an existing
+ * cluster, the cluster's repository version entity must be updated with the custom repos provided by the
+ * management pack. The class takes care of this.
+ */
+public class UpdateActiveRepoVersionOnStartup {
+
+ private static final Logger LOG = LoggerFactory.getLogger(UpdateActiveRepoVersionOnStartup.class);
+
+
+ ClusterDAO clusterDao;
+ ClusterVersionDAO clusterVersionDao;
+ RepositoryVersionDAO repositoryVersionDao;
+ RepositoryVersionHelper repositoryVersionHelper;
+ StackManager stackManager;
+
+
+ private static final Function<RepositoryEntity, String> REPO_TO_ID = new Function<RepositoryEntity, String>() {
+ @Override public String apply(@Nullable RepositoryEntity input) { return input.getRepositoryId(); }
+ };
+
+ @Inject
+ public UpdateActiveRepoVersionOnStartup(ClusterDAO clusterDao,
+ ClusterVersionDAO clusterVersionDao,
+ RepositoryVersionDAO repositoryVersionDao,
+ RepositoryVersionHelper repositoryVersionHelper,
+ AmbariMetaInfo metaInfo) {
+ this.clusterDao = clusterDao;
+ this.clusterVersionDao = clusterVersionDao;
+ this.repositoryVersionDao = repositoryVersionDao;
+ this.repositoryVersionHelper = repositoryVersionHelper;
+ this.stackManager = metaInfo.getStackManager();
+ }
+
+ /**
+ * Updates the active {@link RepositoryVersionEntity} for clusters with add-on services defined in management packs.
+ * @throws AmbariException
+ */
+ @Transactional
+ public void process() throws AmbariException {
+ LOG.info("Updating existing repo versions with service repos.");
+ try {
+ List<ClusterEntity> clusters = clusterDao.findAll();
+ for (ClusterEntity cluster: clusters) {
+ StackInfo stack =
+ stackManager.getStack(cluster.getDesiredStack().getStackName(), cluster.getDesiredStack().getStackVersion());
+ LOG.info("Updating existing repo versions for cluster {} on stack {}-{}",
+ cluster.getClusterName(), stack.getName(), stack.getVersion());
+ ClusterVersionEntity clusterVersion = clusterVersionDao.findByClusterAndStateCurrent(cluster.getClusterName());
+ RepositoryVersionEntity repoVersion = clusterVersion.getRepositoryVersion();
+ updateRepoVersion(stack, repoVersion);
+ repositoryVersionDao.merge(repoVersion);
+ }
+ }
+ catch(Exception ex) {
+ throw new AmbariException(
+ "An error occured during updating current repository versions with stack repositories.",
+ ex);
+ }
+ }
+
+ private void updateRepoVersion(StackInfo stackInfo, RepositoryVersionEntity repoVersion) throws Exception {
+ ListMultimap<String, RepositoryInfo> serviceReposByOs = stackInfo.getRepositoriesByOs();
+
+ // Update repos in the JSON representation
+ List<OperatingSystemEntity> operatingSystems = repoVersion.getOperatingSystems();
+ RepoUtil.addServiceReposToOperatingSystemEntities(operatingSystems, serviceReposByOs);
+ repoVersion.setOperatingSystems(repositoryVersionHelper.serializeOperatingSystemEntities(operatingSystems));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
index 252592f..0b8cab8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
@@ -18,8 +18,12 @@
package org.apache.ambari.server.state;
+import com.google.common.base.Objects;
import org.apache.ambari.server.controller.RepositoryResponse;
+import com.google.common.base.Function;
+import com.google.common.base.Strings;
+
public class RepositoryInfo {
private String baseUrl;
private String osType;
@@ -155,6 +159,24 @@ public class RepositoryInfo {
+ " ]";
}
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ RepositoryInfo that = (RepositoryInfo) o;
+ return Objects.equal(baseUrl, that.baseUrl) &&
+ Objects.equal(osType, that.osType) &&
+ Objects.equal(repoId, that.repoId) &&
+ Objects.equal(repoName, that.repoName) &&
+ Objects.equal(mirrorsList, that.mirrorsList) &&
+ Objects.equal(defaultBaseUrl, that.defaultBaseUrl) &&
+ Objects.equal(latestBaseUrl, that.latestBaseUrl);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(baseUrl, osType, repoId, repoName, mirrorsList, defaultBaseUrl, latestBaseUrl);
+ }
public RepositoryResponse convertToResponse()
{
@@ -162,6 +184,41 @@ public class RepositoryInfo {
getRepoName(), getMirrorsList(), getDefaultBaseUrl(), getLatestBaseUrl());
}
+ /**
+ * A function that returns the repo name of any RepositoryInfo
+ */
+ public static final Function<RepositoryInfo, String> GET_REPO_NAME_FUNCTION = new Function<RepositoryInfo, String>() {
+ @Override public String apply(RepositoryInfo input) {
+ return input.repoName;
+ }
+ };
+
+ /**
+ * A function that returns the repoId of any RepositoryInfo
+ */
+ public static final Function<RepositoryInfo, String> GET_REPO_ID_FUNCTION = new Function<RepositoryInfo, String>() {
+ @Override public String apply(RepositoryInfo input) {
+ return input.repoId;
+ }
+ };
+
+ /**
+ * A function that returns the baseUrl of any RepositoryInfo
+ */
+ public static final Function<RepositoryInfo, String> SAFE_GET_BASE_URL_FUNCTION = new Function<RepositoryInfo, String>() {
+ @Override public String apply(RepositoryInfo input) {
+ return Strings.nullToEmpty(input.baseUrl);
+ }
+ };
+
+ /**
+ * A function that returns the osType of any RepositoryInfo
+ */
+ public static final Function<RepositoryInfo, String> GET_OSTYPE_FUNCTION = new Function<RepositoryInfo, String>() {
+ @Override public String apply(RepositoryInfo input) {
+ return input.osType;
+ }
+ };
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
index 14ff9de..ba5cb42 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
@@ -29,7 +29,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import com.google.common.io.Files;
import org.apache.ambari.server.controller.StackVersionResponse;
import org.apache.ambari.server.stack.Validable;
import org.apache.ambari.server.state.repository.VersionDefinitionXml;
@@ -38,6 +37,12 @@ import org.apache.ambari.server.state.stack.RepositoryXml;
import org.apache.ambari.server.state.stack.StackRoleCommandOrder;
import org.apache.ambari.server.state.stack.UpgradePack;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimaps;
+import com.google.common.io.Files;
+
public class StackInfo implements Comparable<StackInfo>, Validable{
private String minJdk;
private String maxJdk;
@@ -143,6 +148,12 @@ public class StackInfo implements Comparable<StackInfo>, Validable{
return repositories;
}
+ /**
+ * @return A list containing all repos for this stack, grouped by os
+ */
+ public ListMultimap<String, RepositoryInfo> getRepositoriesByOs() {
+ return Multimaps.index(getRepositories(), RepositoryInfo.GET_OSTYPE_FUNCTION);
+ }
public synchronized Collection<ServiceInfo> getServices() {
if (services == null) services = new ArrayList<ServiceInfo>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
index 6cec6b0..9ca6cf4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
@@ -100,13 +100,14 @@ public class RepositoryVersionHelper {
OperatingSystemResourceProvider.OPERATING_SYSTEM_AMBARI_MANAGED_REPOS).getAsBoolean());
}
- for (JsonElement repositoryJson: osObj.get(RepositoryVersionResourceProvider.SUBRESOURCE_REPOSITORIES_PROPERTY_ID).getAsJsonArray()) {
+ for (JsonElement repositoryElement: osObj.get(RepositoryVersionResourceProvider.SUBRESOURCE_REPOSITORIES_PROPERTY_ID).getAsJsonArray()) {
final RepositoryEntity repositoryEntity = new RepositoryEntity();
- repositoryEntity.setBaseUrl(repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_BASE_URL_PROPERTY_ID).getAsString());
- repositoryEntity.setName(repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_REPO_NAME_PROPERTY_ID).getAsString());
- repositoryEntity.setRepositoryId(repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_REPO_ID_PROPERTY_ID).getAsString());
- if (repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID) != null) {
- repositoryEntity.setUnique(repositoryJson.getAsJsonObject().get(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID).getAsBoolean());
+ final JsonObject repositoryJson = repositoryElement.getAsJsonObject();
+ repositoryEntity.setBaseUrl(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_BASE_URL_PROPERTY_ID).getAsString());
+ repositoryEntity.setName(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_REPO_NAME_PROPERTY_ID).getAsString());
+ repositoryEntity.setRepositoryId(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_REPO_ID_PROPERTY_ID).getAsString());
+ if (repositoryJson.get(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID) != null) {
+ repositoryEntity.setUnique(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID).getAsBoolean());
}
operatingSystemEntity.getRepositories().add(repositoryEntity);
}
@@ -165,6 +166,21 @@ public class RepositoryVersionHelper {
return gson.toJson(rootJson);
}
+ public String serializeOperatingSystemEntities(List<OperatingSystemEntity> operatingSystems) {
+ List<RepositoryInfo> repositoryInfos = new ArrayList<>();
+ for (OperatingSystemEntity os: operatingSystems) {
+ for (RepositoryEntity repositoryEntity: os.getRepositories()) {
+ RepositoryInfo repositoryInfo = new RepositoryInfo();
+ repositoryInfo.setRepoId(repositoryEntity.getRepositoryId());
+ repositoryInfo.setRepoName(repositoryEntity.getName());
+ repositoryInfo.setBaseUrl(repositoryEntity.getBaseUrl());
+ repositoryInfo.setOsType(os.getOsType());
+ repositoryInfos.add(repositoryInfo);
+ }
+ }
+ return serializeOperatingSystems(repositoryInfos);
+ }
+
/**
* Scans the given stack for upgrade packages which can be applied to update the cluster to given repository version.
*
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/main/resources/version_definition.xsd
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/version_definition.xsd b/ambari-server/src/main/resources/version_definition.xsd
index 35deb6e..bef3739 100644
--- a/ambari-server/src/main/resources/version_definition.xsd
+++ b/ambari-server/src/main/resources/version_definition.xsd
@@ -24,7 +24,7 @@
xmllint --noout --load-trace --schema [path-to-this-file] [path-to-xml]
</xs:documentation>
</xs:annotation>
-
+
<xs:complexType name="release-type">
<xs:all>
<xs:element name="type" type="repo-type" />
@@ -38,7 +38,7 @@
<xs:element name="package-version" type="xs:string" minOccurs="0" />
</xs:all>
</xs:complexType>
-
+
<xs:simpleType name="repo-type">
<xs:restriction base="xs:string">
<xs:enumeration value="STANDARD" />
@@ -46,7 +46,7 @@
<xs:enumeration value="PATCH" />
</xs:restriction>
</xs:simpleType>
-
+
<xs:simpleType name="family-type">
<xs:restriction base="xs:string">
<xs:enumeration value="redhat6" />
@@ -60,7 +60,7 @@
<xs:enumeration value="suse12" />
</xs:restriction>
</xs:simpleType>
-
+
<xs:complexType name="manifest-service-type">
<xs:annotation>
<xs:documentation>
@@ -86,16 +86,16 @@
</xs:element>
</xs:sequence>
</xs:complexType>
-
+
<xs:complexType name="available-services-type">
<xs:annotation>
<xs:documentation>
Provides a list of services that are available to upgrade out of this repository.
A service may include a list of components that can be upgraded. These are specified
(generally) for patch upgrades only.
-
+
A service must have an 'idref' attribute to tie it back to a service and version from
- the 'manifest' element.
+ the 'manifest' element.
</xs:documentation>
</xs:annotation>
<xs:sequence>
@@ -109,7 +109,7 @@
</xs:element>
</xs:sequence>
</xs:complexType>
-
+
<xs:complexType name="repository-info-type">
<xs:sequence>
<xs:element name="os" maxOccurs="unbounded">
@@ -132,7 +132,7 @@
</xs:element>
</xs:sequence>
</xs:complexType>
-
+
<xs:complexType name="upgrade-type">
<xs:sequence>
<xs:element name="configuration" maxOccurs="unbounded">
@@ -150,7 +150,7 @@
</xs:element>
</xs:sequence>
</xs:complexType>
-
+
<xs:element name="repository-version">
<xs:annotation>
<xs:documentation>
@@ -177,7 +177,7 @@
<xs:selector xpath="./manifest/service" />
<xs:field xpath="@id" />
</xs:key>
-
+
<xs:keyref name="available-services-id-keyref" refer="service-id-key">
<xs:annotation>
<xs:documentation>
@@ -189,5 +189,5 @@
</xs:keyref>
</xs:element>
-
+
</xs:schema>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/java/org/apache/ambari/server/stack/RepoUtilTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/RepoUtilTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/RepoUtilTest.java
new file mode 100644
index 0000000..99a34f4
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/RepoUtilTest.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.stack;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Multimaps;
+import org.apache.ambari.server.controller.RepositoryResponse;
+import org.apache.ambari.server.orm.entities.OperatingSystemEntity;
+import org.apache.ambari.server.orm.entities.RepositoryEntity;
+import org.apache.ambari.server.state.RepositoryInfo;
+import org.apache.ambari.server.state.stack.RepositoryXml;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+
+public class RepoUtilTest {
+
+ private static final List<String> OPERATING_SYSTEMS = ImmutableList.of("redhat6", "sles11", "ubuntu12");
+
+
+ @Test public void testAddServiceReposToOperatingSystemEntities_SimpleCase() {
+ List<OperatingSystemEntity> operatingSystems = new ArrayList<>();
+ for (String os: OPERATING_SYSTEMS) {
+ RepositoryEntity repo1 = repoEntity("HDP", "HDP-2.3", "http://hdp.org/2.3");
+ RepositoryEntity repo2 = repoEntity("HDP-UTILS", "HDP-UTILS-1.1.0", "http://hdp.org/utils/1.1.0");
+ operatingSystems.add(osEntity(os, repo1, repo2));
+ }
+ ListMultimap<String, RepositoryInfo> serviceRepos = serviceRepos(ImmutableList.of("redhat5", "redhat6", "sles11"),
+ "MSFT_R", "MSFT_R-8.1", "http://msft.r");
+
+ RepoUtil.addServiceReposToOperatingSystemEntities(operatingSystems, serviceRepos);
+
+ // Verify results. Service repos should be added only to redhat6 and sles11
+ for (OperatingSystemEntity os: operatingSystems) {
+ Assert.assertNotSame("Redhat5 should not be added as new operating system.", "redhat5", os.getOsType());
+ Optional<RepositoryEntity> msft_r = findRepoEntityById(os.getRepositories(), "MSFT_R-8.1");
+ Assert.assertTrue(
+ String.format("Only redhat6 and sles11 should contain the service repo. os: %s, repo: %s", os.getOsType(), msft_r),
+ findRepoEntityById(os.getRepositories(), "MSFT_R-8.1").isPresent() == ImmutableList.of("redhat6", "sles11").contains(os.getOsType())) ;
+ }
+ }
+
+ @Test public void testAddServiceReposToOperatingSystemEntities_RepoAlreadExists() {
+ List<OperatingSystemEntity> operatingSystems = new ArrayList<>();
+ for (String os: OPERATING_SYSTEMS) {
+ RepositoryEntity repo1 = repoEntity("HDP", "HDP-2.3", "http://hdp.org/2.3");
+ RepositoryEntity repo2 = repoEntity("HDP-UTILS", "HDP-UTILS-1.1.0", "http://hdp.org/utils/1.1.0");
+ RepositoryEntity repo3 = repoEntity("MSFT_R", "MSFT_R-8.1", "http://msft.r.ORIGINAL");
+ operatingSystems.add(osEntity(os, repo1, repo2, repo3));
+ }
+ ListMultimap<String, RepositoryInfo> serviceRepos = serviceRepos(ImmutableList.of("redhat6"),
+ "MSFT_R", "MSFT_R-8.2", "http://msft.r.NEW");
+
+ RepoUtil.addServiceReposToOperatingSystemEntities(operatingSystems, serviceRepos);
+
+ // Verify results. Service repo should not be added second time.
+ for (OperatingSystemEntity os: operatingSystems) {
+ Optional<RepositoryEntity> msft_r_orig = findRepoEntityById(os.getRepositories(), "MSFT_R-8.1");
+ Optional<RepositoryEntity> msft_r_new = findRepoEntityById(os.getRepositories(), "MSFT_R-8.2");
+ Assert.assertTrue("Original repo is missing", msft_r_orig.isPresent());
+ Assert.assertTrue("Service repo with duplicate name should not have been added", !msft_r_new.isPresent());
+ }
+ }
+
+ @Test public void testGetServiceRepos() {
+ List<RepositoryInfo> vdfRepos = Lists.newArrayList(repoInfo("HDP", "HDP-2.3", "redhat6"),
+ repoInfo("HDP-UTILS", "HDP-UTILS-1.1.0.20", "redhat6"),
+ repoInfo("HDP", "HDP-2.3", "redhat5"),
+ repoInfo("HDP-UTILS", "HDP-UTILS-1.1.0.20", "redhat5"));
+ List<RepositoryInfo> stackRepos = Lists.newArrayList(vdfRepos);
+ stackRepos.add(repoInfo("MSFT_R", "MSFT_R-8.1", "redhat6"));
+
+ ImmutableListMultimap<String, RepositoryInfo> stackReposByOs =
+ Multimaps.index(stackRepos, RepositoryInfo.GET_OSTYPE_FUNCTION);
+
+ List<RepositoryInfo> serviceRepos = RepoUtil.getServiceRepos(vdfRepos, stackReposByOs);
+ Assert.assertEquals("Expected 1 service repo", 1, serviceRepos.size());
+ Assert.assertEquals("Expected MSFT_R service repo", "MSFT_R", serviceRepos.get(0).getRepoName());
+ }
+
+ @Test public void testAsRepositoryResponses() {
+ List<RepositoryInfo> repos = Lists.newArrayList(repoInfo("HDP", "HDP-2.3", "redhat6"),
+ repoInfo("HDP-UTILS", "HDP-UTILS-1.1.0.20", "redhat6"),
+ repoInfo("HDP", "HDP-2.3", "redhat5"),
+ repoInfo("HDP-UTILS", "HDP-UTILS-1.1.0.20", "redhat5"));
+ List<RepositoryResponse> responses = RepoUtil.asResponses(repos, "HDP-2.3", "HDP", "2.3");
+
+ Assert.assertEquals("Wrong number of responses", repos.size(), responses.size());
+ for (RepositoryResponse response: responses) {
+ Assert.assertEquals("Unexpected version definition id", "HDP-2.3", response.getVersionDefinitionId());
+ Assert.assertEquals("Unexpected stack name", "HDP", response.getStackName());
+ Assert.assertEquals("Unexpected stack version", "2.3", response.getStackVersion());
+ }
+ }
+
+ private static Optional<RepositoryEntity> findRepoEntityById(Iterable<RepositoryEntity> repos, String repoId) {
+ for (RepositoryEntity repo: repos) if (Objects.equals(repo.getRepositoryId(), repoId)) {
+ return Optional.of(repo);
+ }
+ return Optional.absent();
+ }
+
+ private static OperatingSystemEntity osEntity(String os, RepositoryEntity... repoEntities) {
+ OperatingSystemEntity entity = new OperatingSystemEntity();
+ entity.setOsType(os);
+ for (RepositoryEntity repo: repoEntities) {
+ entity.getRepositories().add(repo);
+ }
+ return entity;
+ }
+
+ private static RepositoryEntity repoEntity(String name, String repoId, String baseUrl) {
+ RepositoryEntity repo = new RepositoryEntity();
+ repo.setName(name);
+ repo.setRepositoryId(repoId);
+ repo.setBaseUrl(baseUrl);
+ return repo;
+ }
+
+ private static RepositoryInfo repoInfo(String name, String repoId, String osType) {
+ RepositoryInfo repo = new RepositoryInfo();
+ repo.setRepoName(name);
+ repo.setRepoId(repoId);
+ repo.setOsType(osType);
+ return repo;
+ }
+
+ private static ListMultimap<String, RepositoryInfo> serviceRepos(List<String> operatingSystems,
+ String repoName, String repoId, String baseUrl) {
+ ArrayListMultimap multimap = ArrayListMultimap.create();
+ for (String os: operatingSystems) {
+ RepositoryInfo repoInfo = new RepositoryInfo();
+ repoInfo.setOsType(os);
+ repoInfo.setRepoId(repoId);
+ repoInfo.setRepoName(repoName);
+ repoInfo.setBaseUrl(baseUrl);
+ multimap.put(os, repoInfo);
+ }
+ return multimap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java
index 1d73ff3..6503e7f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java
@@ -46,6 +46,7 @@ import org.apache.ambari.server.orm.entities.StackEntity;
import org.apache.ambari.server.state.CommandScriptDefinition;
import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.PropertyInfo;
+import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.ServiceOsSpecific;
import org.apache.ambari.server.state.StackInfo;
@@ -55,6 +56,10 @@ import org.easymock.EasyMock;
import org.junit.BeforeClass;
import org.junit.Test;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+
+
/**
* StackManager unit tests.
*/
@@ -141,6 +146,21 @@ public class StackManagerCommonServicesTest {
}
@Test
+ public void testAddOnServiceRepoIsLoaded() {
+ Collection<StackInfo> stacks = stackManager.getStacks("HDP");
+ StackInfo stack = null;
+ for(StackInfo stackInfo: stackManager.getStacks()) {
+ if ("0.2".equals(stackInfo.getVersion())) {
+ stack = stackInfo;
+ break;
+ }
+ }
+ List<RepositoryInfo> repos = stack.getRepositoriesByOs().get("redhat6");
+ ImmutableSet<String> repoIds = ImmutableSet.copyOf(Lists.transform(repos, RepositoryInfo.GET_REPO_ID_FUNCTION));
+ assertTrue("Repos are expected to contain MSFT_R-8.1", repoIds.contains("ADDON_REPO-1.0"));
+ }
+
+ @Test
public void testGetStack() {
StackInfo stack = stackManager.getStack("HDP", "0.1");
assertNotNull(stack);
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/java/org/apache/ambari/server/stack/StackModuleTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackModuleTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackModuleTest.java
new file mode 100644
index 0000000..0b7d0ff
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackModuleTest.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.stack;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.state.RepositoryInfo;
+import org.apache.ambari.server.state.ServiceInfo;
+import org.apache.ambari.server.state.stack.RepositoryXml;
+import org.apache.ambari.server.state.stack.ServiceMetainfoXml;
+import org.junit.Test;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultiset;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multiset;
+
+
+/**
+ * Tests for StackModule
+ */
+public class StackModuleTest {
+
+ @Test
+ public void stackServiceReposAreRead() throws Exception {
+ StackModule sm = createStackModule("FooBar",
+ "2.4",
+ Optional.of(Lists.newArrayList(repoInfo("foo", "1.0.1", "http://foo.org"))),
+ Lists.newArrayList(repoInfo("bar", "2.0.1", "http://bar.org")));
+ Set<String> repoIds = getIds(sm.getModuleInfo().getRepositories());
+ assertEquals(ImmutableSet.of("foo:1.0.1", "bar:2.0.1"), repoIds);
+ }
+
+ /**
+ * If more add-on services define the same repo, the duplicate repo definitions should be disregarded.
+ * @throws Exception
+ */
+ @Test
+ public void duplicateStackServiceReposAreDiscarded() throws Exception {
+ StackModule sm = createStackModule("FooBar",
+ "2.4",
+ // stack repos
+ Optional.of(Lists.newArrayList(repoInfo("StackRepoA", "1.1.1", "http://repos.org/stackrepoA"),
+ repoInfo("StackRepoB", "2.2.2", "http://repos.org/stackrepoB"))),
+
+ // stack service repos
+ // These two should be preserved. even though duplicates, the contents are the same
+ Lists.newArrayList(repoInfo("serviceRepoA", "1.0.0", "http://bar.org/1_0_0")),
+ Lists.newArrayList(repoInfo("serviceRepoA", "1.0.0", "http://bar.org/1_0_0")),
+ // These should be dropped as the names are the same but contents are different
+ Lists.newArrayList(repoInfo("serviceRepoB", "1.2.1", "http://bar.org/1_1_1")),
+ Lists.newArrayList(repoInfo("serviceRepoB", "1.2.3", "http://bar.org/1_1_1")),
+ // The first one should be dropped (overrides a stack repo), the rest only generates warnings (duplicate urls)
+ Lists.newArrayList(repoInfo("StackRepoA", "2.0.0", "http://repos.org/stackrepoA_200"),
+ repoInfo("ShouldBeJustAWarning1", "3.1.1", "http://repos.org/stackrepoA"),
+ repoInfo("ShouldBeJustAWarning2", "1.0.0", "http://bar.org/1_0_0")));
+ List<RepositoryInfo> repos = sm.getModuleInfo().getRepositories();
+
+ Set<String> repoIds = getIds(repos);
+ assertEquals("Unexpected number of repos. Each repo should be added only once", repoIds.size(), repos.size());
+ assertEquals("Unexpected repositories",
+ ImmutableSet.of("StackRepoA:1.1.1",
+ "StackRepoB:2.2.2",
+ "serviceRepoA:1.0.0",
+ "ShouldBeJustAWarning1:3.1.1",
+ "ShouldBeJustAWarning2:1.0.0"), repoIds);
+ }
+
+ @Test
+ public void serviceReposAreProcessedEvenIfNoStackRepo() throws Exception {
+ StackModule sm = createStackModule("FooBar",
+ "2.4",
+ Optional.<List<RepositoryInfo>>absent(),
+ Lists.newArrayList(repoInfo("bar", "2.0.1", "http://bar.org")));
+ Set<String> repoIds = getIds(sm.getModuleInfo().getRepositories());
+ assertEquals(ImmutableSet.of("bar:2.0.1"), repoIds);
+ }
+
+ /**
+ * If two add-on services define the same repo, the repo should be disregarded.
+ * This applies per os, so the same repo can be defined for multiple os'es (e.g redhat5 and redhat6)
+ * @throws Exception
+ */
+ @Test
+ public void duplicateStackServiceReposAreCheckedPerOs() throws Exception {
+ StackModule sm = createStackModule("FooBar",
+ "2.4",
+ Optional.<List<RepositoryInfo>>absent(),
+ Lists.newArrayList(repoInfo("bar", "2.0.1", "http://bar.org", "centos6")),
+ Lists.newArrayList(repoInfo("bar", "2.0.1", "http://bar.org", "centos7")));
+ Multiset<String> repoIds = getIdsMultiple(sm.getModuleInfo().getRepositories());
+ assertEquals("Repo should be occur exactly twice, once for each os type.",
+ ImmutableMultiset.of("bar:2.0.1", "bar:2.0.1"), repoIds);
+ }
+
+ private StackModule createStackModule(String stackName, String stackVersion, Optional<? extends List<RepositoryInfo>> stackRepos,
+ List<RepositoryInfo>... serviceRepoLists) throws AmbariException {
+ StackDirectory sd = mock(StackDirectory.class);
+ List<ServiceDirectory> serviceDirectories = Lists.newArrayList();
+ for (List<RepositoryInfo> serviceRepoList: serviceRepoLists) {
+ StackServiceDirectory svd = mock(StackServiceDirectory.class);
+ RepositoryXml serviceRepoXml = mock(RepositoryXml.class);
+ when(svd.getRepoFile()).thenReturn(serviceRepoXml);
+ when(serviceRepoXml.getRepositories()).thenReturn(serviceRepoList);
+ ServiceMetainfoXml serviceMetainfoXml = mock(ServiceMetainfoXml.class);
+ when(serviceMetainfoXml.isValid()).thenReturn(true);
+ ServiceInfo serviceInfo = mock(ServiceInfo.class);
+ when(serviceInfo.isValid()).thenReturn(true);
+ when(serviceInfo.getName()).thenReturn(UUID.randomUUID().toString()); // unique service names
+ when(serviceMetainfoXml.getServices()).thenReturn(Lists.<ServiceInfo>newArrayList(serviceInfo));
+ when(svd.getMetaInfoFile()).thenReturn(serviceMetainfoXml);
+ serviceDirectories.add(svd);
+ }
+ if (stackRepos.isPresent()) {
+ RepositoryXml stackRepoXml = mock(RepositoryXml.class);
+ when(sd.getRepoFile()).thenReturn(stackRepoXml);
+ when(stackRepoXml.getRepositories()).thenReturn(stackRepos.get());
+ }
+ when(sd.getServiceDirectories()).thenReturn(serviceDirectories);
+ when(sd.getStackDirName()).thenReturn(stackName);
+ when(sd.getDirectory()).thenReturn(new File(stackVersion));
+ StackContext ctx = mock(StackContext.class);
+ StackModule sm = new StackModule(sd, ctx);
+ sm.resolve(null,
+ ImmutableMap.of(String.format("%s:%s", stackName, stackVersion), sm),
+ ImmutableMap.<String, ServiceModule>of(), ImmutableMap.<String, ExtensionModule>of());
+ return sm;
+ }
+
+ private RepositoryInfo repoInfo(String repoName, String repoVersion, String url) {
+ return repoInfo(repoName, repoVersion, url, "centos6");
+ }
+
+ private List<RepositoryInfo> repoInfosForAllOs(String repoName, String repoVersion, String url) {
+ List<RepositoryInfo> repos = new ArrayList<>(3);
+ for (String os: new String[]{ "centos5", "centos6", "centos7"}) {
+ repos.add(repoInfo(repoName, repoVersion, url, os));
+ }
+ return repos;
+ }
+
+
+ private RepositoryInfo repoInfo(String repoName, String repoVersion, String url, String osType) {
+ RepositoryInfo info = new RepositoryInfo();
+ info.setRepoId(String.format("%s:%s", repoName, repoVersion));
+ info.setRepoName(repoName);
+ info.setBaseUrl(url);
+ info.setOsType(osType);
+ return info;
+ }
+
+ private Set<String> getIds(List<RepositoryInfo> repoInfos) {
+ return ImmutableSet.copyOf(Lists.transform(repoInfos, RepositoryInfo.GET_REPO_ID_FUNCTION));
+ }
+
+ private Multiset<String> getIdsMultiple(List<RepositoryInfo> repoInfos) {
+ return ImmutableMultiset.copyOf(Lists.transform(repoInfos, RepositoryInfo.GET_REPO_ID_FUNCTION));
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java
new file mode 100644
index 0000000..9c54a88
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.stack;
+
+
+import static org.mockito.Mockito.*;
+
+import java.io.*;
+
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
+import org.apache.ambari.server.orm.dao.ClusterDAO;
+import org.apache.ambari.server.orm.dao.ClusterVersionDAO;
+import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
+import org.apache.ambari.server.orm.entities.ClusterEntity;
+import org.apache.ambari.server.orm.entities.ClusterVersionEntity;
+import org.apache.ambari.server.orm.entities.OperatingSystemEntity;
+import org.apache.ambari.server.orm.entities.RepositoryEntity;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
+import org.apache.ambari.server.orm.entities.StackEntity;
+import org.apache.ambari.server.state.RepositoryInfo;
+import org.apache.ambari.server.state.StackInfo;
+import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.Resources;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Provider;
+
+/**
+ * Unit test for {@link UpdateActiveRepoVersionOnStartup}
+ */
+public class UpdateActiveRepoVersionOnStartupTest {
+
+ private static String CLUSTER_NAME = "c1";
+ private static String ADD_ON_REPO_ID = "MSFT_R-8.0";
+
+ private RepositoryVersionDAO repositoryVersionDao;
+ private RepositoryVersionEntity repoVersion;
+ private UpdateActiveRepoVersionOnStartup activeRepoUpdater;
+
+ @Test
+ public void addAServiceRepoToExistingRepoVersion() throws Exception {
+ activeRepoUpdater.process();
+ verifyRepoIsAdded();
+ }
+
+ /**
+ * Verifies if the add-on service repo is added to the repo version entity, both json and xml representations.
+ *
+ * @throws Exception
+ */
+ private void verifyRepoIsAdded() throws Exception {
+ verify(repositoryVersionDao, times(1)).merge(repoVersion);
+
+ boolean serviceRepoAddedToJson = false;
+ outer:
+ for (OperatingSystemEntity os: repoVersion.getOperatingSystems()) if (os.getOsType().equals("redhat6")) {
+ for (RepositoryEntity repo: os.getRepositories()) if (repo.getRepositoryId().equals(ADD_ON_REPO_ID)) {
+ serviceRepoAddedToJson = true;
+ break outer;
+ }
+ }
+ Assert.assertTrue(ADD_ON_REPO_ID + " is add-on repo was not added to JSON representation", serviceRepoAddedToJson);
+ }
+
+ @Before
+ public void init() throws Exception {
+ ClusterDAO clusterDao = mock(ClusterDAO.class);
+ ClusterVersionDAO clusterVersionDAO = mock(ClusterVersionDAO.class);
+ repositoryVersionDao = mock(RepositoryVersionDAO.class);
+ final RepositoryVersionHelper repositoryVersionHelper = new RepositoryVersionHelper();
+ AmbariMetaInfo metaInfo = mock(AmbariMetaInfo.class);
+
+ StackManager stackManager = mock(StackManager.class);
+ when(metaInfo.getStackManager()).thenReturn(stackManager);
+
+ ClusterEntity cluster = new ClusterEntity();
+ cluster.setClusterName(CLUSTER_NAME);
+ when(clusterDao.findAll()).thenReturn(ImmutableList.of(cluster));
+
+ StackEntity stackEntity = new StackEntity();
+ stackEntity.setStackName("HDP");
+ stackEntity.setStackVersion("2.3");
+ cluster.setDesiredStack(stackEntity);
+
+ StackInfo stackInfo = new StackInfo();
+ stackInfo.setName("HDP");
+ stackInfo.setVersion("2.3");
+ RepositoryInfo repositoryInfo = new RepositoryInfo();
+ repositoryInfo.setBaseUrl("http://msft.r");
+ repositoryInfo.setRepoId(ADD_ON_REPO_ID);
+ repositoryInfo.setRepoName("MSFT_R");
+ repositoryInfo.setOsType("redhat6");
+ stackInfo.getRepositories().add(repositoryInfo);
+ when(stackManager.getStack("HDP", "2.3")).thenReturn(stackInfo);
+
+ Provider<RepositoryVersionHelper> repositoryVersionHelperProvider = mock(Provider.class);
+ when(repositoryVersionHelperProvider.get()).thenReturn(repositoryVersionHelper);
+ InMemoryDefaultTestModule testModule = new InMemoryDefaultTestModule() {
+ @Override
+ protected void configure() {
+ bind(RepositoryVersionHelper.class).toInstance(repositoryVersionHelper);
+ requestStaticInjection(RepositoryVersionEntity.class);
+ }
+ };
+ Injector injector = Guice.createInjector(testModule);
+ repoVersion = new RepositoryVersionEntity();
+ repoVersion.setStack(stackEntity);
+ repoVersion.setOperatingSystems(resourceAsString("org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest_initialRepos.json"));
+ ClusterVersionEntity clusterVersion = new ClusterVersionEntity();
+ clusterVersion.setRepositoryVersion(repoVersion);
+ when(clusterVersionDAO.findByClusterAndStateCurrent(CLUSTER_NAME)).thenReturn(clusterVersion);
+
+ activeRepoUpdater = new UpdateActiveRepoVersionOnStartup(clusterDao,
+ clusterVersionDAO, repositoryVersionDao, repositoryVersionHelper, metaInfo);
+ }
+
+ private static String resourceAsString(String resourceName) throws IOException {
+ return Resources.toString(Resources.getResource(resourceName), Charsets.UTF_8);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/resources/common-services/ADDON/1.0/configuration/addon-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/common-services/ADDON/1.0/configuration/addon-env.xml b/ambari-server/src/test/resources/common-services/ADDON/1.0/configuration/addon-env.xml
new file mode 100644
index 0000000..7005e68
--- /dev/null
+++ b/ambari-server/src/test/resources/common-services/ADDON/1.0/configuration/addon-env.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<!-- This is a special config file for properties used to monitor status of the service -->
+<configuration supports_adding_forbidden="true">
+ <property>
+ <name>Foo</name>
+ <display-name>Foo property</display-name>
+ <description>Foo property</description>
+ <value>bar</value>
+ <value-attributes>
+ <type>string</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/resources/common-services/ADDON/1.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/common-services/ADDON/1.0/metainfo.xml b/ambari-server/src/test/resources/common-services/ADDON/1.0/metainfo.xml
new file mode 100644
index 0000000..e7fe568
--- /dev/null
+++ b/ambari-server/src/test/resources/common-services/ADDON/1.0/metainfo.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ADDON</name>
+ <version>1.0</version>
+ </service>
+ <components>
+ <component>
+ <name>ADDON_CLIENT</name>
+ <displayName>Add-on service client</displayName>
+ <category>CLIENT</category>
+ <cardinality>1+</cardinality>
+ </component>
+ </components>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/resources/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest_initialRepos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest_initialRepos.json b/ambari-server/src/test/resources/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest_initialRepos.json
new file mode 100644
index 0000000..f59544f
--- /dev/null
+++ b/ambari-server/src/test/resources/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest_initialRepos.json
@@ -0,0 +1,32 @@
+[
+ {
+ "repositories": [
+ {
+ "Repositories/base_url": "http://192.168.99.100/repos/HDP-2.4.0.0/",
+ "Repositories/repo_name": "HDP",
+ "Repositories/repo_id": "HDP-2.4"
+ },
+ {
+ "Repositories/base_url": "http://192.168.99.100/repos/HDP-UTILS-1.1.0.20/",
+ "Repositories/repo_name": "HDP-UTILS",
+ "Repositories/repo_id": "HDP-UTILS-1.1.0.20"
+ }
+ ],
+ "OperatingSystems/os_type": "redhat6"
+ },
+ {
+ "repositories": [
+ {
+ "Repositories/base_url": "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos7/2.x/BUILDS/2.4.3.0-207",
+ "Repositories/repo_name": "HDP",
+ "Repositories/repo_id": "HDP-2.4"
+ },
+ {
+ "Repositories/base_url": "http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos7",
+ "Repositories/repo_name": "HDP-UTILS",
+ "Repositories/repo_id": "HDP-UTILS-1.1.0.20"
+ }
+ ],
+ "OperatingSystems/os_type": "redhat7"
+ }
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/7961cd11/ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/ADDON/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/ADDON/metainfo.xml b/ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/ADDON/metainfo.xml
new file mode 100644
index 0000000..07242db
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks_with_common_services/HDP/0.2/services/ADDON/metainfo.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>ADDON</name>
+ <version>1.0</version>
+ <extends>common-services/ADDON/1.0</extends>
+ </service>
+ </services>
+</metainfo>